Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Dynamic DMA mapping support.
4 *
5 * This implementation is a fallback for platforms that do not support
6 * I/O TLBs (aka DMA address translation hardware).
7 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
8 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
9 * Copyright (C) 2000, 2003 Hewlett-Packard Co
10 * David Mosberger-Tang <davidm@hpl.hp.com>
11 *
12 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
13 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
14 * unnecessary i-cache flushing.
15 * 04/07/.. ak Better overflow handling. Assorted fixes.
16 * 05/09/10 linville Add support for syncing ranges, support syncing for
17 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
18 * 08/12/11 beckyb Add highmem support
19 */
20
21#define pr_fmt(fmt) "software IO TLB: " fmt
22
23#include <linux/cache.h>
24#include <linux/cc_platform.h>
25#include <linux/ctype.h>
26#include <linux/debugfs.h>
27#include <linux/dma-direct.h>
28#include <linux/dma-map-ops.h>
29#include <linux/export.h>
30#include <linux/gfp.h>
31#include <linux/highmem.h>
32#include <linux/io.h>
33#include <linux/iommu-helper.h>
34#include <linux/init.h>
35#include <linux/memblock.h>
36#include <linux/mm.h>
37#include <linux/pfn.h>
38#include <linux/rculist.h>
39#include <linux/scatterlist.h>
40#include <linux/set_memory.h>
41#include <linux/spinlock.h>
42#include <linux/string.h>
43#include <linux/swiotlb.h>
44#include <linux/types.h>
45#ifdef CONFIG_DMA_RESTRICTED_POOL
46#include <linux/of.h>
47#include <linux/of_fdt.h>
48#include <linux/of_reserved_mem.h>
49#include <linux/slab.h>
50#endif
51
52#define CREATE_TRACE_POINTS
53#include <trace/events/swiotlb.h>
54
55#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
56
57/*
58 * Minimum IO TLB size to bother booting with. Systems with mainly
59 * 64bit capable cards will only lightly use the swiotlb. If we can't
60 * allocate a contiguous 1MB, we're probably in trouble anyway.
61 */
62#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
63
64#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
65
66/**
67 * struct io_tlb_slot - IO TLB slot descriptor
68 * @orig_addr: The original address corresponding to a mapped entry.
69 * @alloc_size: Size of the allocated buffer.
70 * @list: The free list describing the number of free entries available
71 * from each index.
72 * @pad_slots: Number of preceding padding slots. Valid only in the first
73 * allocated non-padding slot.
74 */
75struct io_tlb_slot {
76 phys_addr_t orig_addr;
77 size_t alloc_size;
78 unsigned short list;
79 unsigned short pad_slots;
80};
81
82static bool swiotlb_force_bounce;
83static bool swiotlb_force_disable;
84
85#ifdef CONFIG_SWIOTLB_DYNAMIC
86
87static void swiotlb_dyn_alloc(struct work_struct *work);
88
89static struct io_tlb_mem io_tlb_default_mem = {
90 .lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
91 .pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
92 .dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
93 swiotlb_dyn_alloc),
94};
95
96#else /* !CONFIG_SWIOTLB_DYNAMIC */
97
98static struct io_tlb_mem io_tlb_default_mem;
99
100#endif /* CONFIG_SWIOTLB_DYNAMIC */
101
102static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
103static unsigned long default_nareas;
104
105/**
106 * struct io_tlb_area - IO TLB memory area descriptor
107 *
108 * This is a single area with a single lock.
109 *
110 * @used: The number of used IO TLB block.
111 * @index: The slot index to start searching in this area for next round.
112 * @lock: The lock to protect the above data structures in the map and
113 * unmap calls.
114 */
115struct io_tlb_area {
116 unsigned long used;
117 unsigned int index;
118 spinlock_t lock;
119};
120
121/*
122 * Round up number of slabs to the next power of 2. The last area is going
123 * be smaller than the rest if default_nslabs is not power of two.
124 * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
125 * otherwise a segment may span two or more areas. It conflicts with free
126 * contiguous slots tracking: free slots are treated contiguous no matter
127 * whether they cross an area boundary.
128 *
129 * Return true if default_nslabs is rounded up.
130 */
131static bool round_up_default_nslabs(void)
132{
133 if (!default_nareas)
134 return false;
135
136 if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
137 default_nslabs = IO_TLB_SEGSIZE * default_nareas;
138 else if (is_power_of_2(default_nslabs))
139 return false;
140 default_nslabs = roundup_pow_of_two(default_nslabs);
141 return true;
142}
143
144/**
145 * swiotlb_adjust_nareas() - adjust the number of areas and slots
146 * @nareas: Desired number of areas. Zero is treated as 1.
147 *
148 * Adjust the default number of areas in a memory pool.
149 * The default size of the memory pool may also change to meet minimum area
150 * size requirements.
151 */
152static void swiotlb_adjust_nareas(unsigned int nareas)
153{
154 if (!nareas)
155 nareas = 1;
156 else if (!is_power_of_2(nareas))
157 nareas = roundup_pow_of_two(nareas);
158
159 default_nareas = nareas;
160
161 pr_info("area num %d.\n", nareas);
162 if (round_up_default_nslabs())
163 pr_info("SWIOTLB bounce buffer size roundup to %luMB",
164 (default_nslabs << IO_TLB_SHIFT) >> 20);
165}
166
167/**
168 * limit_nareas() - get the maximum number of areas for a given memory pool size
169 * @nareas: Desired number of areas.
170 * @nslots: Total number of slots in the memory pool.
171 *
172 * Limit the number of areas to the maximum possible number of areas in
173 * a memory pool of the given size.
174 *
175 * Return: Maximum possible number of areas.
176 */
177static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
178{
179 if (nslots < nareas * IO_TLB_SEGSIZE)
180 return nslots / IO_TLB_SEGSIZE;
181 return nareas;
182}
183
184static int __init
185setup_io_tlb_npages(char *str)
186{
187 if (isdigit(*str)) {
188 /* avoid tail segment of size < IO_TLB_SEGSIZE */
189 default_nslabs =
190 ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
191 }
192 if (*str == ',')
193 ++str;
194 if (isdigit(*str))
195 swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));
196 if (*str == ',')
197 ++str;
198 if (!strcmp(str, "force"))
199 swiotlb_force_bounce = true;
200 else if (!strcmp(str, "noforce"))
201 swiotlb_force_disable = true;
202
203 return 0;
204}
205early_param("swiotlb", setup_io_tlb_npages);
206
207unsigned long swiotlb_size_or_default(void)
208{
209 return default_nslabs << IO_TLB_SHIFT;
210}
211
212void __init swiotlb_adjust_size(unsigned long size)
213{
214 /*
215 * If swiotlb parameter has not been specified, give a chance to
216 * architectures such as those supporting memory encryption to
217 * adjust/expand SWIOTLB size for their use.
218 */
219 if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
220 return;
221
222 size = ALIGN(size, IO_TLB_SIZE);
223 default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
224 if (round_up_default_nslabs())
225 size = default_nslabs << IO_TLB_SHIFT;
226 pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
227}
228
229void swiotlb_print_info(void)
230{
231 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
232
233 if (!mem->nslabs) {
234 pr_warn("No low mem\n");
235 return;
236 }
237
238 pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
239 (mem->nslabs << IO_TLB_SHIFT) >> 20);
240}
241
242static inline unsigned long io_tlb_offset(unsigned long val)
243{
244 return val & (IO_TLB_SEGSIZE - 1);
245}
246
247static inline unsigned long nr_slots(u64 val)
248{
249 return DIV_ROUND_UP(val, IO_TLB_SIZE);
250}
251
252/*
253 * Early SWIOTLB allocation may be too early to allow an architecture to
254 * perform the desired operations. This function allows the architecture to
255 * call SWIOTLB when the operations are possible. It needs to be called
256 * before the SWIOTLB memory is used.
257 */
258void __init swiotlb_update_mem_attributes(void)
259{
260 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
261 unsigned long bytes;
262
263 if (!mem->nslabs || mem->late_alloc)
264 return;
265 bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
266 set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
267}
268
269static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
270 unsigned long nslabs, bool late_alloc, unsigned int nareas)
271{
272 void *vaddr = phys_to_virt(start);
273 unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
274
275 mem->nslabs = nslabs;
276 mem->start = start;
277 mem->end = mem->start + bytes;
278 mem->late_alloc = late_alloc;
279 mem->nareas = nareas;
280 mem->area_nslabs = nslabs / mem->nareas;
281
282 for (i = 0; i < mem->nareas; i++) {
283 spin_lock_init(&mem->areas[i].lock);
284 mem->areas[i].index = 0;
285 mem->areas[i].used = 0;
286 }
287
288 for (i = 0; i < mem->nslabs; i++) {
289 mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),
290 mem->nslabs - i);
291 mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
292 mem->slots[i].alloc_size = 0;
293 mem->slots[i].pad_slots = 0;
294 }
295
296 memset(vaddr, 0, bytes);
297 mem->vaddr = vaddr;
298 return;
299}
300
301/**
302 * add_mem_pool() - add a memory pool to the allocator
303 * @mem: Software IO TLB allocator.
304 * @pool: Memory pool to be added.
305 */
306static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
307{
308#ifdef CONFIG_SWIOTLB_DYNAMIC
309 spin_lock(&mem->lock);
310 list_add_rcu(&pool->node, &mem->pools);
311 mem->nslabs += pool->nslabs;
312 spin_unlock(&mem->lock);
313#else
314 mem->nslabs = pool->nslabs;
315#endif
316}
317
318static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
319 unsigned int flags,
320 int (*remap)(void *tlb, unsigned long nslabs))
321{
322 size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
323 void *tlb;
324
325 /*
326 * By default allocate the bounce buffer memory from low memory, but
327 * allow to pick a location everywhere for hypervisors with guest
328 * memory encryption.
329 */
330 if (flags & SWIOTLB_ANY)
331 tlb = memblock_alloc(bytes, PAGE_SIZE);
332 else
333 tlb = memblock_alloc_low(bytes, PAGE_SIZE);
334
335 if (!tlb) {
336 pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
337 __func__, bytes);
338 return NULL;
339 }
340
341 if (remap && remap(tlb, nslabs) < 0) {
342 memblock_free(tlb, PAGE_ALIGN(bytes));
343 pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
344 return NULL;
345 }
346
347 return tlb;
348}
349
350/*
351 * Statically reserve bounce buffer space and initialize bounce buffer data
352 * structures for the software IO TLB used to implement the DMA API.
353 */
354void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
355 int (*remap)(void *tlb, unsigned long nslabs))
356{
357 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
358 unsigned long nslabs;
359 unsigned int nareas;
360 size_t alloc_size;
361 void *tlb;
362
363 if (!addressing_limit && !swiotlb_force_bounce)
364 return;
365 if (swiotlb_force_disable)
366 return;
367
368 io_tlb_default_mem.force_bounce =
369 swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
370
371#ifdef CONFIG_SWIOTLB_DYNAMIC
372 if (!remap)
373 io_tlb_default_mem.can_grow = true;
374 if (flags & SWIOTLB_ANY)
375 io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
376 else
377 io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
378#endif
379
380 if (!default_nareas)
381 swiotlb_adjust_nareas(num_possible_cpus());
382
383 nslabs = default_nslabs;
384 nareas = limit_nareas(default_nareas, nslabs);
385 while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
386 if (nslabs <= IO_TLB_MIN_SLABS)
387 return;
388 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
389 nareas = limit_nareas(nareas, nslabs);
390 }
391
392 if (default_nslabs != nslabs) {
393 pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
394 default_nslabs, nslabs);
395 default_nslabs = nslabs;
396 }
397
398 alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
399 mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
400 if (!mem->slots) {
401 pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
402 __func__, alloc_size, PAGE_SIZE);
403 return;
404 }
405
406 mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
407 nareas), SMP_CACHE_BYTES);
408 if (!mem->areas) {
409 pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
410 return;
411 }
412
413 swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas);
414 add_mem_pool(&io_tlb_default_mem, mem);
415
416 if (flags & SWIOTLB_VERBOSE)
417 swiotlb_print_info();
418}
419
420void __init swiotlb_init(bool addressing_limit, unsigned int flags)
421{
422 swiotlb_init_remap(addressing_limit, flags, NULL);
423}
424
425/*
426 * Systems with larger DMA zones (those that don't support ISA) can
427 * initialize the swiotlb later using the slab allocator if needed.
428 * This should be just like above, but with some error catching.
429 */
430int swiotlb_init_late(size_t size, gfp_t gfp_mask,
431 int (*remap)(void *tlb, unsigned long nslabs))
432{
433 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
434 unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
435 unsigned int nareas;
436 unsigned char *vstart = NULL;
437 unsigned int order, area_order;
438 bool retried = false;
439 int rc = 0;
440
441 if (io_tlb_default_mem.nslabs)
442 return 0;
443
444 if (swiotlb_force_disable)
445 return 0;
446
447 io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
448
449#ifdef CONFIG_SWIOTLB_DYNAMIC
450 if (!remap)
451 io_tlb_default_mem.can_grow = true;
452 if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
453 io_tlb_default_mem.phys_limit = zone_dma_limit;
454 else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
455 io_tlb_default_mem.phys_limit = max(DMA_BIT_MASK(32), zone_dma_limit);
456 else
457 io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
458#endif
459
460 if (!default_nareas)
461 swiotlb_adjust_nareas(num_possible_cpus());
462
463retry:
464 order = get_order(nslabs << IO_TLB_SHIFT);
465 nslabs = SLABS_PER_PAGE << order;
466
467 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
468 vstart = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
469 order);
470 if (vstart)
471 break;
472 order--;
473 nslabs = SLABS_PER_PAGE << order;
474 retried = true;
475 }
476
477 if (!vstart)
478 return -ENOMEM;
479
480 if (remap)
481 rc = remap(vstart, nslabs);
482 if (rc) {
483 free_pages((unsigned long)vstart, order);
484
485 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
486 if (nslabs < IO_TLB_MIN_SLABS)
487 return rc;
488 retried = true;
489 goto retry;
490 }
491
492 if (retried) {
493 pr_warn("only able to allocate %ld MB\n",
494 (PAGE_SIZE << order) >> 20);
495 }
496
497 nareas = limit_nareas(default_nareas, nslabs);
498 area_order = get_order(array_size(sizeof(*mem->areas), nareas));
499 mem->areas = (struct io_tlb_area *)
500 __get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
501 if (!mem->areas)
502 goto error_area;
503
504 mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
505 get_order(array_size(sizeof(*mem->slots), nslabs)));
506 if (!mem->slots)
507 goto error_slots;
508
509 set_memory_decrypted((unsigned long)vstart,
510 (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
511 swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
512 nareas);
513 add_mem_pool(&io_tlb_default_mem, mem);
514
515 swiotlb_print_info();
516 return 0;
517
518error_slots:
519 free_pages((unsigned long)mem->areas, area_order);
520error_area:
521 free_pages((unsigned long)vstart, order);
522 return -ENOMEM;
523}
524
525void __init swiotlb_exit(void)
526{
527 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
528 unsigned long tbl_vaddr;
529 size_t tbl_size, slots_size;
530 unsigned int area_order;
531
532 if (swiotlb_force_bounce)
533 return;
534
535 if (!mem->nslabs)
536 return;
537
538 pr_info("tearing down default memory pool\n");
539 tbl_vaddr = (unsigned long)phys_to_virt(mem->start);
540 tbl_size = PAGE_ALIGN(mem->end - mem->start);
541 slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
542
543 set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
544 if (mem->late_alloc) {
545 area_order = get_order(array_size(sizeof(*mem->areas),
546 mem->nareas));
547 free_pages((unsigned long)mem->areas, area_order);
548 free_pages(tbl_vaddr, get_order(tbl_size));
549 free_pages((unsigned long)mem->slots, get_order(slots_size));
550 } else {
551 memblock_free_late(__pa(mem->areas),
552 array_size(sizeof(*mem->areas), mem->nareas));
553 memblock_free_late(mem->start, tbl_size);
554 memblock_free_late(__pa(mem->slots), slots_size);
555 }
556
557 memset(mem, 0, sizeof(*mem));
558}
559
560#ifdef CONFIG_SWIOTLB_DYNAMIC
561
562/**
563 * alloc_dma_pages() - allocate pages to be used for DMA
564 * @gfp: GFP flags for the allocation.
565 * @bytes: Size of the buffer.
566 * @phys_limit: Maximum allowed physical address of the buffer.
567 *
568 * Allocate pages from the buddy allocator. If successful, make the allocated
569 * pages decrypted that they can be used for DMA.
570 *
571 * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
572 * if the allocated physical address was above @phys_limit.
573 */
574static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
575{
576 unsigned int order = get_order(bytes);
577 struct page *page;
578 phys_addr_t paddr;
579 void *vaddr;
580
581 page = alloc_pages(gfp, order);
582 if (!page)
583 return NULL;
584
585 paddr = page_to_phys(page);
586 if (paddr + bytes - 1 > phys_limit) {
587 __free_pages(page, order);
588 return ERR_PTR(-EAGAIN);
589 }
590
591 vaddr = phys_to_virt(paddr);
592 if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
593 goto error;
594 return page;
595
596error:
597 /* Intentional leak if pages cannot be encrypted again. */
598 if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
599 __free_pages(page, order);
600 return NULL;
601}
602
603/**
604 * swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
605 * @dev: Device for which a memory pool is allocated.
606 * @bytes: Size of the buffer.
607 * @phys_limit: Maximum allowed physical address of the buffer.
608 * @gfp: GFP flags for the allocation.
609 *
610 * Return: Allocated pages, or %NULL on allocation failure.
611 */
612static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
613 u64 phys_limit, gfp_t gfp)
614{
615 struct page *page;
616
617 /*
618 * Allocate from the atomic pools if memory is encrypted and
619 * the allocation is atomic, because decrypting may block.
620 */
621 if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
622 void *vaddr;
623
624 if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
625 return NULL;
626
627 return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
628 dma_coherent_ok);
629 }
630
631 gfp &= ~GFP_ZONEMASK;
632 if (phys_limit <= zone_dma_limit)
633 gfp |= __GFP_DMA;
634 else if (phys_limit <= DMA_BIT_MASK(32))
635 gfp |= __GFP_DMA32;
636
637 while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
638 if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
639 phys_limit < DMA_BIT_MASK(64) &&
640 !(gfp & (__GFP_DMA32 | __GFP_DMA)))
641 gfp |= __GFP_DMA32;
642 else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
643 !(gfp & __GFP_DMA))
644 gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
645 else
646 return NULL;
647 }
648
649 return page;
650}
651
652/**
653 * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
654 * @vaddr: Virtual address of the buffer.
655 * @bytes: Size of the buffer.
656 */
657static void swiotlb_free_tlb(void *vaddr, size_t bytes)
658{
659 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
660 dma_free_from_pool(NULL, vaddr, bytes))
661 return;
662
663 /* Intentional leak if pages cannot be encrypted again. */
664 if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
665 __free_pages(virt_to_page(vaddr), get_order(bytes));
666}
667
668/**
669 * swiotlb_alloc_pool() - allocate a new IO TLB memory pool
670 * @dev: Device for which a memory pool is allocated.
671 * @minslabs: Minimum number of slabs.
672 * @nslabs: Desired (maximum) number of slabs.
673 * @nareas: Number of areas.
674 * @phys_limit: Maximum DMA buffer physical address.
675 * @gfp: GFP flags for the allocations.
676 *
677 * Allocate and initialize a new IO TLB memory pool. The actual number of
678 * slabs may be reduced if allocation of @nslabs fails. If even
679 * @minslabs cannot be allocated, this function fails.
680 *
681 * Return: New memory pool, or %NULL on allocation failure.
682 */
683static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
684 unsigned long minslabs, unsigned long nslabs,
685 unsigned int nareas, u64 phys_limit, gfp_t gfp)
686{
687 struct io_tlb_pool *pool;
688 unsigned int slot_order;
689 struct page *tlb;
690 size_t pool_size;
691 size_t tlb_size;
692
693 if (nslabs > SLABS_PER_PAGE << MAX_PAGE_ORDER) {
694 nslabs = SLABS_PER_PAGE << MAX_PAGE_ORDER;
695 nareas = limit_nareas(nareas, nslabs);
696 }
697
698 pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
699 pool = kzalloc(pool_size, gfp);
700 if (!pool)
701 goto error;
702 pool->areas = (void *)pool + sizeof(*pool);
703
704 tlb_size = nslabs << IO_TLB_SHIFT;
705 while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
706 if (nslabs <= minslabs)
707 goto error_tlb;
708 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
709 nareas = limit_nareas(nareas, nslabs);
710 tlb_size = nslabs << IO_TLB_SHIFT;
711 }
712
713 slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
714 pool->slots = (struct io_tlb_slot *)
715 __get_free_pages(gfp, slot_order);
716 if (!pool->slots)
717 goto error_slots;
718
719 swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas);
720 return pool;
721
722error_slots:
723 swiotlb_free_tlb(page_address(tlb), tlb_size);
724error_tlb:
725 kfree(pool);
726error:
727 return NULL;
728}
729
730/**
731 * swiotlb_dyn_alloc() - dynamic memory pool allocation worker
732 * @work: Pointer to dyn_alloc in struct io_tlb_mem.
733 */
734static void swiotlb_dyn_alloc(struct work_struct *work)
735{
736 struct io_tlb_mem *mem =
737 container_of(work, struct io_tlb_mem, dyn_alloc);
738 struct io_tlb_pool *pool;
739
740 pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
741 default_nareas, mem->phys_limit, GFP_KERNEL);
742 if (!pool) {
743 pr_warn_ratelimited("Failed to allocate new pool");
744 return;
745 }
746
747 add_mem_pool(mem, pool);
748}
749
750/**
751 * swiotlb_dyn_free() - RCU callback to free a memory pool
752 * @rcu: RCU head in the corresponding struct io_tlb_pool.
753 */
754static void swiotlb_dyn_free(struct rcu_head *rcu)
755{
756 struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
757 size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
758 size_t tlb_size = pool->end - pool->start;
759
760 free_pages((unsigned long)pool->slots, get_order(slots_size));
761 swiotlb_free_tlb(pool->vaddr, tlb_size);
762 kfree(pool);
763}
764
765/**
766 * __swiotlb_find_pool() - find the IO TLB pool for a physical address
767 * @dev: Device which has mapped the DMA buffer.
768 * @paddr: Physical address within the DMA buffer.
769 *
770 * Find the IO TLB memory pool descriptor which contains the given physical
771 * address, if any. This function is for use only when the dev is known to
772 * be using swiotlb. Use swiotlb_find_pool() for the more general case
773 * when this condition is not met.
774 *
775 * Return: Memory pool which contains @paddr, or %NULL if none.
776 */
777struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
778{
779 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
780 struct io_tlb_pool *pool;
781
782 rcu_read_lock();
783 list_for_each_entry_rcu(pool, &mem->pools, node) {
784 if (paddr >= pool->start && paddr < pool->end)
785 goto out;
786 }
787
788 list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
789 if (paddr >= pool->start && paddr < pool->end)
790 goto out;
791 }
792 pool = NULL;
793out:
794 rcu_read_unlock();
795 return pool;
796}
797
798/**
799 * swiotlb_del_pool() - remove an IO TLB pool from a device
800 * @dev: Owning device.
801 * @pool: Memory pool to be removed.
802 */
803static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
804{
805 unsigned long flags;
806
807 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
808 list_del_rcu(&pool->node);
809 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
810
811 call_rcu(&pool->rcu, swiotlb_dyn_free);
812}
813
814#endif /* CONFIG_SWIOTLB_DYNAMIC */
815
816/**
817 * swiotlb_dev_init() - initialize swiotlb fields in &struct device
818 * @dev: Device to be initialized.
819 */
820void swiotlb_dev_init(struct device *dev)
821{
822 dev->dma_io_tlb_mem = &io_tlb_default_mem;
823#ifdef CONFIG_SWIOTLB_DYNAMIC
824 INIT_LIST_HEAD(&dev->dma_io_tlb_pools);
825 spin_lock_init(&dev->dma_io_tlb_lock);
826 dev->dma_uses_io_tlb = false;
827#endif
828}
829
830/**
831 * swiotlb_align_offset() - Get required offset into an IO TLB allocation.
832 * @dev: Owning device.
833 * @align_mask: Allocation alignment mask.
834 * @addr: DMA address.
835 *
836 * Return the minimum offset from the start of an IO TLB allocation which is
837 * required for a given buffer address and allocation alignment to keep the
838 * device happy.
839 *
840 * First, the address bits covered by min_align_mask must be identical in the
841 * original address and the bounce buffer address. High bits are preserved by
842 * choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
843 * padding bytes before the bounce buffer.
844 *
845 * Second, @align_mask specifies which bits of the first allocated slot must
846 * be zero. This may require allocating additional padding slots, and then the
847 * offset (in bytes) from the first such padding slot is returned.
848 */
849static unsigned int swiotlb_align_offset(struct device *dev,
850 unsigned int align_mask, u64 addr)
851{
852 return addr & dma_get_min_align_mask(dev) &
853 (align_mask | (IO_TLB_SIZE - 1));
854}
855
856/*
857 * Bounce: copy the swiotlb buffer from or back to the original dma location
858 */
859static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
860 enum dma_data_direction dir, struct io_tlb_pool *mem)
861{
862 int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
863 phys_addr_t orig_addr = mem->slots[index].orig_addr;
864 size_t alloc_size = mem->slots[index].alloc_size;
865 unsigned long pfn = PFN_DOWN(orig_addr);
866 unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
867 int tlb_offset;
868
869 if (orig_addr == INVALID_PHYS_ADDR)
870 return;
871
872 /*
873 * It's valid for tlb_offset to be negative. This can happen when the
874 * "offset" returned by swiotlb_align_offset() is non-zero, and the
875 * tlb_addr is pointing within the first "offset" bytes of the second
876 * or subsequent slots of the allocated swiotlb area. While it's not
877 * valid for tlb_addr to be pointing within the first "offset" bytes
878 * of the first slot, there's no way to check for such an error since
879 * this function can't distinguish the first slot from the second and
880 * subsequent slots.
881 */
882 tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
883 swiotlb_align_offset(dev, 0, orig_addr);
884
885 orig_addr += tlb_offset;
886 alloc_size -= tlb_offset;
887
888 if (size > alloc_size) {
889 dev_WARN_ONCE(dev, 1,
890 "Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
891 alloc_size, size);
892 size = alloc_size;
893 }
894
895 if (PageHighMem(pfn_to_page(pfn))) {
896 unsigned int offset = orig_addr & ~PAGE_MASK;
897 struct page *page;
898 unsigned int sz = 0;
899 unsigned long flags;
900
901 while (size) {
902 sz = min_t(size_t, PAGE_SIZE - offset, size);
903
904 local_irq_save(flags);
905 page = pfn_to_page(pfn);
906 if (dir == DMA_TO_DEVICE)
907 memcpy_from_page(vaddr, page, offset, sz);
908 else
909 memcpy_to_page(page, offset, vaddr, sz);
910 local_irq_restore(flags);
911
912 size -= sz;
913 pfn++;
914 vaddr += sz;
915 offset = 0;
916 }
917 } else if (dir == DMA_TO_DEVICE) {
918 memcpy(vaddr, phys_to_virt(orig_addr), size);
919 } else {
920 memcpy(phys_to_virt(orig_addr), vaddr, size);
921 }
922}
923
924static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
925{
926 return start + (idx << IO_TLB_SHIFT);
927}
928
929/*
930 * Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
931 */
932static inline unsigned long get_max_slots(unsigned long boundary_mask)
933{
934 return (boundary_mask >> IO_TLB_SHIFT) + 1;
935}
936
937static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
938{
939 if (index >= mem->area_nslabs)
940 return 0;
941 return index;
942}
943
944/*
945 * Track the total used slots with a global atomic value in order to have
946 * correct information to determine the high water mark. The mem_used()
947 * function gives imprecise results because there's no locking across
948 * multiple areas.
949 */
950#ifdef CONFIG_DEBUG_FS
951static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
952{
953 unsigned long old_hiwater, new_used;
954
955 new_used = atomic_long_add_return(nslots, &mem->total_used);
956 old_hiwater = atomic_long_read(&mem->used_hiwater);
957 do {
958 if (new_used <= old_hiwater)
959 break;
960 } while (!atomic_long_try_cmpxchg(&mem->used_hiwater,
961 &old_hiwater, new_used));
962}
963
964static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
965{
966 atomic_long_sub(nslots, &mem->total_used);
967}
968
969#else /* !CONFIG_DEBUG_FS */
970static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
971{
972}
973static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
974{
975}
976#endif /* CONFIG_DEBUG_FS */
977
978#ifdef CONFIG_SWIOTLB_DYNAMIC
979#ifdef CONFIG_DEBUG_FS
980static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
981{
982 atomic_long_add(nslots, &mem->transient_nslabs);
983}
984
985static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
986{
987 atomic_long_sub(nslots, &mem->transient_nslabs);
988}
989
990#else /* !CONFIG_DEBUG_FS */
991static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
992{
993}
994static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
995{
996}
997#endif /* CONFIG_DEBUG_FS */
998#endif /* CONFIG_SWIOTLB_DYNAMIC */
999
1000/**
1001 * swiotlb_search_pool_area() - search one memory area in one pool
1002 * @dev: Device which maps the buffer.
1003 * @pool: Memory pool to be searched.
1004 * @area_index: Index of the IO TLB memory area to be searched.
1005 * @orig_addr: Original (non-bounced) IO buffer address.
1006 * @alloc_size: Total requested size of the bounce buffer,
1007 * including initial alignment padding.
1008 * @alloc_align_mask: Required alignment of the allocated buffer.
1009 *
1010 * Find a suitable sequence of IO TLB entries for the request and allocate
1011 * a buffer from the given IO TLB memory area.
1012 * This function takes care of locking.
1013 *
1014 * Return: Index of the first allocated slot, or -1 on error.
1015 */
1016static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
1017 int area_index, phys_addr_t orig_addr, size_t alloc_size,
1018 unsigned int alloc_align_mask)
1019{
1020 struct io_tlb_area *area = pool->areas + area_index;
1021 unsigned long boundary_mask = dma_get_seg_boundary(dev);
1022 dma_addr_t tbl_dma_addr =
1023 phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
1024 unsigned long max_slots = get_max_slots(boundary_mask);
1025 unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
1026 unsigned int nslots = nr_slots(alloc_size), stride;
1027 unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);
1028 unsigned int index, slots_checked, count = 0, i;
1029 unsigned long flags;
1030 unsigned int slot_base;
1031 unsigned int slot_index;
1032
1033 BUG_ON(!nslots);
1034 BUG_ON(area_index >= pool->nareas);
1035
1036 /*
1037 * Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
1038 * page-aligned in the absence of any other alignment requirements.
1039 * 'alloc_align_mask' was later introduced to specify the alignment
1040 * explicitly, however this is passed as zero for streaming mappings
1041 * and so we preserve the old behaviour there in case any drivers are
1042 * relying on it.
1043 */
1044 if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
1045 alloc_align_mask = PAGE_SIZE - 1;
1046
1047 /*
1048 * Ensure that the allocation is at least slot-aligned and update
1049 * 'iotlb_align_mask' to ignore bits that will be preserved when
1050 * offsetting into the allocation.
1051 */
1052 alloc_align_mask |= (IO_TLB_SIZE - 1);
1053 iotlb_align_mask &= ~alloc_align_mask;
1054
1055 /*
1056 * For mappings with an alignment requirement don't bother looping to
1057 * unaligned slots once we found an aligned one.
1058 */
1059 stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
1060
1061 spin_lock_irqsave(&area->lock, flags);
1062 if (unlikely(nslots > pool->area_nslabs - area->used))
1063 goto not_found;
1064
1065 slot_base = area_index * pool->area_nslabs;
1066 index = area->index;
1067
1068 for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1069 phys_addr_t tlb_addr;
1070
1071 slot_index = slot_base + index;
1072 tlb_addr = slot_addr(tbl_dma_addr, slot_index);
1073
1074 if ((tlb_addr & alloc_align_mask) ||
1075 (orig_addr && (tlb_addr & iotlb_align_mask) !=
1076 (orig_addr & iotlb_align_mask))) {
1077 index = wrap_area_index(pool, index + 1);
1078 slots_checked++;
1079 continue;
1080 }
1081
1082 if (!iommu_is_span_boundary(slot_index, nslots,
1083 nr_slots(tbl_dma_addr),
1084 max_slots)) {
1085 if (pool->slots[slot_index].list >= nslots)
1086 goto found;
1087 }
1088 index = wrap_area_index(pool, index + stride);
1089 slots_checked += stride;
1090 }
1091
1092not_found:
1093 spin_unlock_irqrestore(&area->lock, flags);
1094 return -1;
1095
1096found:
1097 /*
1098 * If we find a slot that indicates we have 'nslots' number of
1099 * contiguous buffers, we allocate the buffers from that slot onwards
1100 * and set the list of free entries to '0' indicating unavailable.
1101 */
1102 for (i = slot_index; i < slot_index + nslots; i++) {
1103 pool->slots[i].list = 0;
1104 pool->slots[i].alloc_size = alloc_size - (offset +
1105 ((i - slot_index) << IO_TLB_SHIFT));
1106 }
1107 for (i = slot_index - 1;
1108 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
1109 pool->slots[i].list; i--)
1110 pool->slots[i].list = ++count;
1111
1112 /*
1113 * Update the indices to avoid searching in the next round.
1114 */
1115 area->index = wrap_area_index(pool, index + nslots);
1116 area->used += nslots;
1117 spin_unlock_irqrestore(&area->lock, flags);
1118
1119 inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots);
1120 return slot_index;
1121}
1122
1123#ifdef CONFIG_SWIOTLB_DYNAMIC
1124
1125/**
1126 * swiotlb_search_area() - search one memory area in all pools
1127 * @dev: Device which maps the buffer.
1128 * @start_cpu: Start CPU number.
1129 * @cpu_offset: Offset from @start_cpu.
1130 * @orig_addr: Original (non-bounced) IO buffer address.
1131 * @alloc_size: Total requested size of the bounce buffer,
1132 * including initial alignment padding.
1133 * @alloc_align_mask: Required alignment of the allocated buffer.
1134 * @retpool: Used memory pool, updated on return.
1135 *
1136 * Search one memory area in all pools for a sequence of slots that match the
1137 * allocation constraints.
1138 *
1139 * Return: Index of the first allocated slot, or -1 on error.
1140 */
1141static int swiotlb_search_area(struct device *dev, int start_cpu,
1142 int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1143 unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1144{
1145 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1146 struct io_tlb_pool *pool;
1147 int area_index;
1148 int index = -1;
1149
1150 rcu_read_lock();
1151 list_for_each_entry_rcu(pool, &mem->pools, node) {
1152 if (cpu_offset >= pool->nareas)
1153 continue;
1154 area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1155 index = swiotlb_search_pool_area(dev, pool, area_index,
1156 orig_addr, alloc_size,
1157 alloc_align_mask);
1158 if (index >= 0) {
1159 *retpool = pool;
1160 break;
1161 }
1162 }
1163 rcu_read_unlock();
1164 return index;
1165}
1166
1167/**
1168 * swiotlb_find_slots() - search for slots in the whole swiotlb
1169 * @dev: Device which maps the buffer.
1170 * @orig_addr: Original (non-bounced) IO buffer address.
1171 * @alloc_size: Total requested size of the bounce buffer,
1172 * including initial alignment padding.
1173 * @alloc_align_mask: Required alignment of the allocated buffer.
1174 * @retpool: Used memory pool, updated on return.
1175 *
1176 * Search through the whole software IO TLB to find a sequence of slots that
1177 * match the allocation constraints.
1178 *
1179 * Return: Index of the first allocated slot, or -1 on error.
1180 */
1181static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1182 size_t alloc_size, unsigned int alloc_align_mask,
1183 struct io_tlb_pool **retpool)
1184{
1185 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1186 struct io_tlb_pool *pool;
1187 unsigned long nslabs;
1188 unsigned long flags;
1189 u64 phys_limit;
1190 int cpu, i;
1191 int index;
1192
1193 if (alloc_size > IO_TLB_SEGSIZE * IO_TLB_SIZE)
1194 return -1;
1195
1196 cpu = raw_smp_processor_id();
1197 for (i = 0; i < default_nareas; ++i) {
1198 index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,
1199 alloc_align_mask, &pool);
1200 if (index >= 0)
1201 goto found;
1202 }
1203
1204 if (!mem->can_grow)
1205 return -1;
1206
1207 schedule_work(&mem->dyn_alloc);
1208
1209 nslabs = nr_slots(alloc_size);
1210 phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
1211 pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
1212 GFP_NOWAIT | __GFP_NOWARN);
1213 if (!pool)
1214 return -1;
1215
1216 index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
1217 alloc_size, alloc_align_mask);
1218 if (index < 0) {
1219 swiotlb_dyn_free(&pool->rcu);
1220 return -1;
1221 }
1222
1223 pool->transient = true;
1224 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
1225 list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);
1226 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
1227 inc_transient_used(mem, pool->nslabs);
1228
1229found:
1230 WRITE_ONCE(dev->dma_uses_io_tlb, true);
1231
1232 /*
1233 * The general barrier orders reads and writes against a presumed store
1234 * of the SWIOTLB buffer address by a device driver (to a driver private
1235 * data structure). It serves two purposes.
1236 *
1237 * First, the store to dev->dma_uses_io_tlb must be ordered before the
1238 * presumed store. This guarantees that the returned buffer address
1239 * cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
1240 *
1241 * Second, the load from mem->pools must be ordered before the same
1242 * presumed store. This guarantees that the returned buffer address
1243 * cannot be observed by another CPU before an update of the RCU list
1244 * that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1245 * atomicity).
1246 *
1247 * See also the comment in swiotlb_find_pool().
1248 */
1249 smp_mb();
1250
1251 *retpool = pool;
1252 return index;
1253}
1254
1255#else /* !CONFIG_SWIOTLB_DYNAMIC */
1256
1257static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1258 size_t alloc_size, unsigned int alloc_align_mask,
1259 struct io_tlb_pool **retpool)
1260{
1261 struct io_tlb_pool *pool;
1262 int start, i;
1263 int index;
1264
1265 *retpool = pool = &dev->dma_io_tlb_mem->defpool;
1266 i = start = raw_smp_processor_id() & (pool->nareas - 1);
1267 do {
1268 index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1269 alloc_size, alloc_align_mask);
1270 if (index >= 0)
1271 return index;
1272 if (++i >= pool->nareas)
1273 i = 0;
1274 } while (i != start);
1275 return -1;
1276}
1277
1278#endif /* CONFIG_SWIOTLB_DYNAMIC */
1279
1280#ifdef CONFIG_DEBUG_FS
1281
1282/**
1283 * mem_used() - get number of used slots in an allocator
1284 * @mem: Software IO TLB allocator.
1285 *
1286 * The result is accurate in this version of the function, because an atomic
1287 * counter is available if CONFIG_DEBUG_FS is set.
1288 *
1289 * Return: Number of used slots.
1290 */
1291static unsigned long mem_used(struct io_tlb_mem *mem)
1292{
1293 return atomic_long_read(&mem->total_used);
1294}
1295
1296#else /* !CONFIG_DEBUG_FS */
1297
1298/**
1299 * mem_pool_used() - get number of used slots in a memory pool
1300 * @pool: Software IO TLB memory pool.
1301 *
1302 * The result is not accurate, see mem_used().
1303 *
1304 * Return: Approximate number of used slots.
1305 */
1306static unsigned long mem_pool_used(struct io_tlb_pool *pool)
1307{
1308 int i;
1309 unsigned long used = 0;
1310
1311 for (i = 0; i < pool->nareas; i++)
1312 used += pool->areas[i].used;
1313 return used;
1314}
1315
1316/**
1317 * mem_used() - get number of used slots in an allocator
1318 * @mem: Software IO TLB allocator.
1319 *
1320 * The result is not accurate, because there is no locking of individual
1321 * areas.
1322 *
1323 * Return: Approximate number of used slots.
1324 */
1325static unsigned long mem_used(struct io_tlb_mem *mem)
1326{
1327#ifdef CONFIG_SWIOTLB_DYNAMIC
1328 struct io_tlb_pool *pool;
1329 unsigned long used = 0;
1330
1331 rcu_read_lock();
1332 list_for_each_entry_rcu(pool, &mem->pools, node)
1333 used += mem_pool_used(pool);
1334 rcu_read_unlock();
1335
1336 return used;
1337#else
1338 return mem_pool_used(&mem->defpool);
1339#endif
1340}
1341
1342#endif /* CONFIG_DEBUG_FS */
1343
1344/**
1345 * swiotlb_tbl_map_single() - bounce buffer map a single contiguous physical area
1346 * @dev: Device which maps the buffer.
1347 * @orig_addr: Original (non-bounced) physical IO buffer address
1348 * @mapping_size: Requested size of the actual bounce buffer, excluding
1349 * any pre- or post-padding for alignment
1350 * @alloc_align_mask: Required start and end alignment of the allocated buffer
1351 * @dir: DMA direction
1352 * @attrs: Optional DMA attributes for the map operation
1353 *
1354 * Find and allocate a suitable sequence of IO TLB slots for the request.
1355 * The allocated space starts at an alignment specified by alloc_align_mask,
1356 * and the size of the allocated space is rounded up so that the total amount
1357 * of allocated space is a multiple of (alloc_align_mask + 1). If
1358 * alloc_align_mask is zero, the allocated space may be at any alignment and
1359 * the size is not rounded up.
1360 *
1361 * The returned address is within the allocated space and matches the bits
1362 * of orig_addr that are specified in the DMA min_align_mask for the device. As
1363 * such, this returned address may be offset from the beginning of the allocated
1364 * space. The bounce buffer space starting at the returned address for
1365 * mapping_size bytes is initialized to the contents of the original IO buffer
1366 * area. Any pre-padding (due to an offset) and any post-padding (due to
1367 * rounding-up the size) is not initialized.
1368 */
1369phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
1370 size_t mapping_size, unsigned int alloc_align_mask,
1371 enum dma_data_direction dir, unsigned long attrs)
1372{
1373 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1374 unsigned int offset;
1375 struct io_tlb_pool *pool;
1376 unsigned int i;
1377 size_t size;
1378 int index;
1379 phys_addr_t tlb_addr;
1380 unsigned short pad_slots;
1381
1382 if (!mem || !mem->nslabs) {
1383 dev_warn_ratelimited(dev,
1384 "Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
1385 return (phys_addr_t)DMA_MAPPING_ERROR;
1386 }
1387
1388 if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
1389 pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
1390
1391 /*
1392 * The default swiotlb memory pool is allocated with PAGE_SIZE
1393 * alignment. If a mapping is requested with larger alignment,
1394 * the mapping may be unable to use the initial slot(s) in all
1395 * sets of IO_TLB_SEGSIZE slots. In such case, a mapping request
1396 * of or near the maximum mapping size would always fail.
1397 */
1398 dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK,
1399 "Alloc alignment may prevent fulfilling requests with max mapping_size\n");
1400
1401 offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);
1402 size = ALIGN(mapping_size + offset, alloc_align_mask + 1);
1403 index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);
1404 if (index == -1) {
1405 if (!(attrs & DMA_ATTR_NO_WARN))
1406 dev_warn_ratelimited(dev,
1407 "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
1408 size, mem->nslabs, mem_used(mem));
1409 return (phys_addr_t)DMA_MAPPING_ERROR;
1410 }
1411
1412 /*
1413 * If dma_skip_sync was set, reset it on first SWIOTLB buffer
1414 * mapping to always sync SWIOTLB buffers.
1415 */
1416 dma_reset_need_sync(dev);
1417
1418 /*
1419 * Save away the mapping from the original address to the DMA address.
1420 * This is needed when we sync the memory. Then we sync the buffer if
1421 * needed.
1422 */
1423 pad_slots = offset >> IO_TLB_SHIFT;
1424 offset &= (IO_TLB_SIZE - 1);
1425 index += pad_slots;
1426 pool->slots[index].pad_slots = pad_slots;
1427 for (i = 0; i < (nr_slots(size) - pad_slots); i++)
1428 pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
1429 tlb_addr = slot_addr(pool->start, index) + offset;
1430 /*
1431 * When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
1432 * the original buffer to the TLB buffer before initiating DMA in order
1433 * to preserve the original's data if the device does a partial write,
1434 * i.e. if the device doesn't overwrite the entire buffer. Preserving
1435 * the original data, even if it's garbage, is necessary to match
1436 * hardware behavior. Use of swiotlb is supposed to be transparent,
1437 * i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1438 */
1439 swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE, pool);
1440 return tlb_addr;
1441}
1442
1443static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
1444 struct io_tlb_pool *mem)
1445{
1446 unsigned long flags;
1447 unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
1448 int index, nslots, aindex;
1449 struct io_tlb_area *area;
1450 int count, i;
1451
1452 index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
1453 index -= mem->slots[index].pad_slots;
1454 nslots = nr_slots(mem->slots[index].alloc_size + offset);
1455 aindex = index / mem->area_nslabs;
1456 area = &mem->areas[aindex];
1457
1458 /*
1459 * Return the buffer to the free list by setting the corresponding
1460 * entries to indicate the number of contiguous entries available.
1461 * While returning the entries to the free list, we merge the entries
1462 * with slots below and above the pool being returned.
1463 */
1464 BUG_ON(aindex >= mem->nareas);
1465
1466 spin_lock_irqsave(&area->lock, flags);
1467 if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
1468 count = mem->slots[index + nslots].list;
1469 else
1470 count = 0;
1471
1472 /*
1473 * Step 1: return the slots to the free list, merging the slots with
1474 * superceeding slots
1475 */
1476 for (i = index + nslots - 1; i >= index; i--) {
1477 mem->slots[i].list = ++count;
1478 mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
1479 mem->slots[i].alloc_size = 0;
1480 mem->slots[i].pad_slots = 0;
1481 }
1482
1483 /*
1484 * Step 2: merge the returned slots with the preceding slots, if
1485 * available (non zero)
1486 */
1487 for (i = index - 1;
1488 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
1489 i--)
1490 mem->slots[i].list = ++count;
1491 area->used -= nslots;
1492 spin_unlock_irqrestore(&area->lock, flags);
1493
1494 dec_used(dev->dma_io_tlb_mem, nslots);
1495}
1496
1497#ifdef CONFIG_SWIOTLB_DYNAMIC
1498
1499/**
1500 * swiotlb_del_transient() - delete a transient memory pool
1501 * @dev: Device which mapped the buffer.
1502 * @tlb_addr: Physical address within a bounce buffer.
1503 * @pool: Pointer to the transient memory pool to be checked and deleted.
1504 *
1505 * Check whether the address belongs to a transient SWIOTLB memory pool.
1506 * If yes, then delete the pool.
1507 *
1508 * Return: %true if @tlb_addr belonged to a transient pool that was released.
1509 */
1510static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr,
1511 struct io_tlb_pool *pool)
1512{
1513 if (!pool->transient)
1514 return false;
1515
1516 dec_used(dev->dma_io_tlb_mem, pool->nslabs);
1517 swiotlb_del_pool(dev, pool);
1518 dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs);
1519 return true;
1520}
1521
1522#else /* !CONFIG_SWIOTLB_DYNAMIC */
1523
1524static inline bool swiotlb_del_transient(struct device *dev,
1525 phys_addr_t tlb_addr, struct io_tlb_pool *pool)
1526{
1527 return false;
1528}
1529
1530#endif /* CONFIG_SWIOTLB_DYNAMIC */
1531
1532/*
1533 * tlb_addr is the physical address of the bounce buffer to unmap.
1534 */
1535void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1536 size_t mapping_size, enum dma_data_direction dir,
1537 unsigned long attrs, struct io_tlb_pool *pool)
1538{
1539 /*
1540 * First, sync the memory before unmapping the entry
1541 */
1542 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1543 (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1544 swiotlb_bounce(dev, tlb_addr, mapping_size,
1545 DMA_FROM_DEVICE, pool);
1546
1547 if (swiotlb_del_transient(dev, tlb_addr, pool))
1548 return;
1549 swiotlb_release_slots(dev, tlb_addr, pool);
1550}
1551
1552void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1553 size_t size, enum dma_data_direction dir,
1554 struct io_tlb_pool *pool)
1555{
1556 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1557 swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE, pool);
1558 else
1559 BUG_ON(dir != DMA_FROM_DEVICE);
1560}
1561
1562void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1563 size_t size, enum dma_data_direction dir,
1564 struct io_tlb_pool *pool)
1565{
1566 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1567 swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE, pool);
1568 else
1569 BUG_ON(dir != DMA_TO_DEVICE);
1570}
1571
1572/*
1573 * Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
1574 * to the device copy the data into it as well.
1575 */
1576dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
1577 enum dma_data_direction dir, unsigned long attrs)
1578{
1579 phys_addr_t swiotlb_addr;
1580 dma_addr_t dma_addr;
1581
1582 trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
1583
1584 swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
1585 if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
1586 return DMA_MAPPING_ERROR;
1587
1588 /* Ensure that the address returned is DMA'ble */
1589 dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
1590 if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
1591 __swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
1592 attrs | DMA_ATTR_SKIP_CPU_SYNC,
1593 swiotlb_find_pool(dev, swiotlb_addr));
1594 dev_WARN_ONCE(dev, 1,
1595 "swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1596 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
1597 return DMA_MAPPING_ERROR;
1598 }
1599
1600 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1601 arch_sync_dma_for_device(swiotlb_addr, size, dir);
1602 return dma_addr;
1603}
1604
1605size_t swiotlb_max_mapping_size(struct device *dev)
1606{
1607 int min_align_mask = dma_get_min_align_mask(dev);
1608 int min_align = 0;
1609
1610 /*
1611 * swiotlb_find_slots() skips slots according to
1612 * min align mask. This affects max mapping size.
1613 * Take it into acount here.
1614 */
1615 if (min_align_mask)
1616 min_align = roundup(min_align_mask, IO_TLB_SIZE);
1617
1618 return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
1619}
1620
1621/**
1622 * is_swiotlb_allocated() - check if the default software IO TLB is initialized
1623 */
1624bool is_swiotlb_allocated(void)
1625{
1626 return io_tlb_default_mem.nslabs;
1627}
1628
1629bool is_swiotlb_active(struct device *dev)
1630{
1631 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1632
1633 return mem && mem->nslabs;
1634}
1635
1636/**
1637 * default_swiotlb_base() - get the base address of the default SWIOTLB
1638 *
1639 * Get the lowest physical address used by the default software IO TLB pool.
1640 */
1641phys_addr_t default_swiotlb_base(void)
1642{
1643#ifdef CONFIG_SWIOTLB_DYNAMIC
1644 io_tlb_default_mem.can_grow = false;
1645#endif
1646 return io_tlb_default_mem.defpool.start;
1647}
1648
1649/**
1650 * default_swiotlb_limit() - get the address limit of the default SWIOTLB
1651 *
1652 * Get the highest physical address used by the default software IO TLB pool.
1653 */
1654phys_addr_t default_swiotlb_limit(void)
1655{
1656#ifdef CONFIG_SWIOTLB_DYNAMIC
1657 return io_tlb_default_mem.phys_limit;
1658#else
1659 return io_tlb_default_mem.defpool.end - 1;
1660#endif
1661}
1662
1663#ifdef CONFIG_DEBUG_FS
1664#ifdef CONFIG_SWIOTLB_DYNAMIC
1665static unsigned long mem_transient_used(struct io_tlb_mem *mem)
1666{
1667 return atomic_long_read(&mem->transient_nslabs);
1668}
1669
1670static int io_tlb_transient_used_get(void *data, u64 *val)
1671{
1672 struct io_tlb_mem *mem = data;
1673
1674 *val = mem_transient_used(mem);
1675 return 0;
1676}
1677
1678DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get,
1679 NULL, "%llu\n");
1680#endif /* CONFIG_SWIOTLB_DYNAMIC */
1681
1682static int io_tlb_used_get(void *data, u64 *val)
1683{
1684 struct io_tlb_mem *mem = data;
1685
1686 *val = mem_used(mem);
1687 return 0;
1688}
1689
1690static int io_tlb_hiwater_get(void *data, u64 *val)
1691{
1692 struct io_tlb_mem *mem = data;
1693
1694 *val = atomic_long_read(&mem->used_hiwater);
1695 return 0;
1696}
1697
1698static int io_tlb_hiwater_set(void *data, u64 val)
1699{
1700 struct io_tlb_mem *mem = data;
1701
1702 /* Only allow setting to zero */
1703 if (val != 0)
1704 return -EINVAL;
1705
1706 atomic_long_set(&mem->used_hiwater, val);
1707 return 0;
1708}
1709
1710DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
1711DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
1712 io_tlb_hiwater_set, "%llu\n");
1713
1714static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1715 const char *dirname)
1716{
1717 mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
1718 if (!mem->nslabs)
1719 return;
1720
1721 debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
1722 debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
1723 &fops_io_tlb_used);
1724 debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
1725 &fops_io_tlb_hiwater);
1726#ifdef CONFIG_SWIOTLB_DYNAMIC
1727 debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs,
1728 mem, &fops_io_tlb_transient_used);
1729#endif
1730}
1731
1732static int __init swiotlb_create_default_debugfs(void)
1733{
1734 swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb");
1735 return 0;
1736}
1737
1738late_initcall(swiotlb_create_default_debugfs);
1739
1740#else /* !CONFIG_DEBUG_FS */
1741
1742static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1743 const char *dirname)
1744{
1745}
1746
1747#endif /* CONFIG_DEBUG_FS */
1748
1749#ifdef CONFIG_DMA_RESTRICTED_POOL
1750
1751struct page *swiotlb_alloc(struct device *dev, size_t size)
1752{
1753 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1754 struct io_tlb_pool *pool;
1755 phys_addr_t tlb_addr;
1756 unsigned int align;
1757 int index;
1758
1759 if (!mem)
1760 return NULL;
1761
1762 align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
1763 index = swiotlb_find_slots(dev, 0, size, align, &pool);
1764 if (index == -1)
1765 return NULL;
1766
1767 tlb_addr = slot_addr(pool->start, index);
1768 if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1769 dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1770 &tlb_addr);
1771 swiotlb_release_slots(dev, tlb_addr, pool);
1772 return NULL;
1773 }
1774
1775 return pfn_to_page(PFN_DOWN(tlb_addr));
1776}
1777
1778bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1779{
1780 phys_addr_t tlb_addr = page_to_phys(page);
1781 struct io_tlb_pool *pool;
1782
1783 pool = swiotlb_find_pool(dev, tlb_addr);
1784 if (!pool)
1785 return false;
1786
1787 swiotlb_release_slots(dev, tlb_addr, pool);
1788
1789 return true;
1790}
1791
1792static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
1793 struct device *dev)
1794{
1795 struct io_tlb_mem *mem = rmem->priv;
1796 unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
1797
1798 /* Set Per-device io tlb area to one */
1799 unsigned int nareas = 1;
1800
1801 if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
1802 dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
1803 return -EINVAL;
1804 }
1805
1806 /*
1807 * Since multiple devices can share the same pool, the private data,
1808 * io_tlb_mem struct, will be initialized by the first device attached
1809 * to it.
1810 */
1811 if (!mem) {
1812 struct io_tlb_pool *pool;
1813
1814 mem = kzalloc(sizeof(*mem), GFP_KERNEL);
1815 if (!mem)
1816 return -ENOMEM;
1817 pool = &mem->defpool;
1818
1819 pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL);
1820 if (!pool->slots) {
1821 kfree(mem);
1822 return -ENOMEM;
1823 }
1824
1825 pool->areas = kcalloc(nareas, sizeof(*pool->areas),
1826 GFP_KERNEL);
1827 if (!pool->areas) {
1828 kfree(pool->slots);
1829 kfree(mem);
1830 return -ENOMEM;
1831 }
1832
1833 set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
1834 rmem->size >> PAGE_SHIFT);
1835 swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
1836 false, nareas);
1837 mem->force_bounce = true;
1838 mem->for_alloc = true;
1839#ifdef CONFIG_SWIOTLB_DYNAMIC
1840 spin_lock_init(&mem->lock);
1841 INIT_LIST_HEAD_RCU(&mem->pools);
1842#endif
1843 add_mem_pool(mem, pool);
1844
1845 rmem->priv = mem;
1846
1847 swiotlb_create_debugfs_files(mem, rmem->name);
1848 }
1849
1850 dev->dma_io_tlb_mem = mem;
1851
1852 return 0;
1853}
1854
1855static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
1856 struct device *dev)
1857{
1858 dev->dma_io_tlb_mem = &io_tlb_default_mem;
1859}
1860
1861static const struct reserved_mem_ops rmem_swiotlb_ops = {
1862 .device_init = rmem_swiotlb_device_init,
1863 .device_release = rmem_swiotlb_device_release,
1864};
1865
1866static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
1867{
1868 unsigned long node = rmem->fdt_node;
1869
1870 if (of_get_flat_dt_prop(node, "reusable", NULL) ||
1871 of_get_flat_dt_prop(node, "linux,cma-default", NULL) ||
1872 of_get_flat_dt_prop(node, "linux,dma-default", NULL) ||
1873 of_get_flat_dt_prop(node, "no-map", NULL))
1874 return -EINVAL;
1875
1876 rmem->ops = &rmem_swiotlb_ops;
1877 pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
1878 &rmem->base, (unsigned long)rmem->size / SZ_1M);
1879 return 0;
1880}
1881
1882RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
1883#endif /* CONFIG_DMA_RESTRICTED_POOL */