Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Re-map IO memory to kernel address space so that we can access it.
4 * This is needed for high PCI addresses that aren't mapped in the
5 * 640k-1MB IO memory area on PC's
6 *
7 * (C) Copyright 1995 1996 Linus Torvalds
8 */
9
10#include <linux/memblock.h>
11#include <linux/init.h>
12#include <linux/io.h>
13#include <linux/ioport.h>
14#include <linux/slab.h>
15#include <linux/vmalloc.h>
16#include <linux/mmiotrace.h>
17#include <linux/cc_platform.h>
18#include <linux/efi.h>
19#include <linux/pgtable.h>
20#include <linux/kmsan.h>
21
22#include <asm/set_memory.h>
23#include <asm/e820/api.h>
24#include <asm/efi.h>
25#include <asm/fixmap.h>
26#include <asm/tlbflush.h>
27#include <asm/pgalloc.h>
28#include <asm/memtype.h>
29#include <asm/setup.h>
30
31#include "physaddr.h"
32
33/*
34 * Descriptor controlling ioremap() behavior.
35 */
36struct ioremap_desc {
37 unsigned int flags;
38};
39
40/*
41 * Fix up the linear direct mapping of the kernel to avoid cache attribute
42 * conflicts.
43 */
44int ioremap_change_attr(unsigned long vaddr, unsigned long size,
45 enum page_cache_mode pcm)
46{
47 unsigned long nrpages = size >> PAGE_SHIFT;
48 int err;
49
50 switch (pcm) {
51 case _PAGE_CACHE_MODE_UC:
52 default:
53 err = _set_memory_uc(vaddr, nrpages);
54 break;
55 case _PAGE_CACHE_MODE_WC:
56 err = _set_memory_wc(vaddr, nrpages);
57 break;
58 case _PAGE_CACHE_MODE_WT:
59 err = _set_memory_wt(vaddr, nrpages);
60 break;
61 case _PAGE_CACHE_MODE_WB:
62 err = _set_memory_wb(vaddr, nrpages);
63 break;
64 }
65
66 return err;
67}
68
69/* Does the range (or a subset of) contain normal RAM? */
70static unsigned int __ioremap_check_ram(struct resource *res)
71{
72 unsigned long start_pfn, stop_pfn;
73 unsigned long i;
74
75 if ((res->flags & IORESOURCE_SYSTEM_RAM) != IORESOURCE_SYSTEM_RAM)
76 return 0;
77
78 start_pfn = (res->start + PAGE_SIZE - 1) >> PAGE_SHIFT;
79 stop_pfn = (res->end + 1) >> PAGE_SHIFT;
80 if (stop_pfn > start_pfn) {
81 for (i = 0; i < (stop_pfn - start_pfn); ++i)
82 if (pfn_valid(start_pfn + i) &&
83 !PageReserved(pfn_to_page(start_pfn + i)))
84 return IORES_MAP_SYSTEM_RAM;
85 }
86
87 return 0;
88}
89
90/*
91 * In a SEV guest, NONE and RESERVED should not be mapped encrypted because
92 * there the whole memory is already encrypted.
93 */
94static unsigned int __ioremap_check_encrypted(struct resource *res)
95{
96 if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
97 return 0;
98
99 switch (res->desc) {
100 case IORES_DESC_NONE:
101 case IORES_DESC_RESERVED:
102 break;
103 default:
104 return IORES_MAP_ENCRYPTED;
105 }
106
107 return 0;
108}
109
110/*
111 * The EFI runtime services data area is not covered by walk_mem_res(), but must
112 * be mapped encrypted when SEV is active.
113 */
114static void __ioremap_check_other(resource_size_t addr, struct ioremap_desc *desc)
115{
116 if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
117 return;
118
119 if (x86_platform.hyper.is_private_mmio(addr)) {
120 desc->flags |= IORES_MAP_ENCRYPTED;
121 return;
122 }
123
124 if (!IS_ENABLED(CONFIG_EFI))
125 return;
126
127 if (efi_mem_type(addr) == EFI_RUNTIME_SERVICES_DATA ||
128 (efi_mem_type(addr) == EFI_BOOT_SERVICES_DATA &&
129 efi_mem_attributes(addr) & EFI_MEMORY_RUNTIME))
130 desc->flags |= IORES_MAP_ENCRYPTED;
131}
132
133static int __ioremap_collect_map_flags(struct resource *res, void *arg)
134{
135 struct ioremap_desc *desc = arg;
136
137 if (!(desc->flags & IORES_MAP_SYSTEM_RAM))
138 desc->flags |= __ioremap_check_ram(res);
139
140 if (!(desc->flags & IORES_MAP_ENCRYPTED))
141 desc->flags |= __ioremap_check_encrypted(res);
142
143 return ((desc->flags & (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED)) ==
144 (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED));
145}
146
147/*
148 * To avoid multiple resource walks, this function walks resources marked as
149 * IORESOURCE_MEM and IORESOURCE_BUSY and looking for system RAM and/or a
150 * resource described not as IORES_DESC_NONE (e.g. IORES_DESC_ACPI_TABLES).
151 *
152 * After that, deal with misc other ranges in __ioremap_check_other() which do
153 * not fall into the above category.
154 */
155static void __ioremap_check_mem(resource_size_t addr, unsigned long size,
156 struct ioremap_desc *desc)
157{
158 u64 start, end;
159
160 start = (u64)addr;
161 end = start + size - 1;
162 memset(desc, 0, sizeof(struct ioremap_desc));
163
164 walk_mem_res(start, end, desc, __ioremap_collect_map_flags);
165
166 __ioremap_check_other(addr, desc);
167}
168
169/*
170 * Remap an arbitrary physical address space into the kernel virtual
171 * address space. It transparently creates kernel huge I/O mapping when
172 * the physical address is aligned by a huge page size (1GB or 2MB) and
173 * the requested size is at least the huge page size.
174 *
175 * NOTE: MTRRs can override PAT memory types with a 4KB granularity.
176 * Therefore, the mapping code falls back to use a smaller page toward 4KB
177 * when a mapping range is covered by non-WB type of MTRRs.
178 *
179 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
180 * have to convert them into an offset in a page-aligned mapping, but the
181 * caller shouldn't need to know that small detail.
182 */
183static void __iomem *
184__ioremap_caller(resource_size_t phys_addr, unsigned long size,
185 enum page_cache_mode pcm, void *caller, bool encrypted)
186{
187 unsigned long offset, vaddr;
188 resource_size_t last_addr;
189 const resource_size_t unaligned_phys_addr = phys_addr;
190 const unsigned long unaligned_size = size;
191 struct ioremap_desc io_desc;
192 struct vm_struct *area;
193 enum page_cache_mode new_pcm;
194 pgprot_t prot;
195 int retval;
196 void __iomem *ret_addr;
197
198 /* Don't allow wraparound or zero size */
199 last_addr = phys_addr + size - 1;
200 if (!size || last_addr < phys_addr)
201 return NULL;
202
203 if (!phys_addr_valid(phys_addr)) {
204 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
205 (unsigned long long)phys_addr);
206 WARN_ON_ONCE(1);
207 return NULL;
208 }
209
210 __ioremap_check_mem(phys_addr, size, &io_desc);
211
212 /*
213 * Don't allow anybody to remap normal RAM that we're using..
214 */
215 if (io_desc.flags & IORES_MAP_SYSTEM_RAM) {
216 WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n",
217 &phys_addr, &last_addr);
218 return NULL;
219 }
220
221 /*
222 * Mappings have to be page-aligned
223 */
224 offset = phys_addr & ~PAGE_MASK;
225 phys_addr &= PAGE_MASK;
226 size = PAGE_ALIGN(last_addr+1) - phys_addr;
227
228 /*
229 * Mask out any bits not part of the actual physical
230 * address, like memory encryption bits.
231 */
232 phys_addr &= PHYSICAL_PAGE_MASK;
233
234 retval = memtype_reserve(phys_addr, (u64)phys_addr + size,
235 pcm, &new_pcm);
236 if (retval) {
237 printk(KERN_ERR "ioremap memtype_reserve failed %d\n", retval);
238 return NULL;
239 }
240
241 if (pcm != new_pcm) {
242 if (!is_new_memtype_allowed(phys_addr, size, pcm, new_pcm)) {
243 printk(KERN_ERR
244 "ioremap error for 0x%llx-0x%llx, requested 0x%x, got 0x%x\n",
245 (unsigned long long)phys_addr,
246 (unsigned long long)(phys_addr + size),
247 pcm, new_pcm);
248 goto err_free_memtype;
249 }
250 pcm = new_pcm;
251 }
252
253 /*
254 * If the page being mapped is in memory and SEV is active then
255 * make sure the memory encryption attribute is enabled in the
256 * resulting mapping.
257 * In TDX guests, memory is marked private by default. If encryption
258 * is not requested (using encrypted), explicitly set decrypt
259 * attribute in all IOREMAPPED memory.
260 */
261 prot = PAGE_KERNEL_IO;
262 if ((io_desc.flags & IORES_MAP_ENCRYPTED) || encrypted)
263 prot = pgprot_encrypted(prot);
264 else
265 prot = pgprot_decrypted(prot);
266
267 switch (pcm) {
268 case _PAGE_CACHE_MODE_UC:
269 default:
270 prot = __pgprot(pgprot_val(prot) |
271 cachemode2protval(_PAGE_CACHE_MODE_UC));
272 break;
273 case _PAGE_CACHE_MODE_UC_MINUS:
274 prot = __pgprot(pgprot_val(prot) |
275 cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS));
276 break;
277 case _PAGE_CACHE_MODE_WC:
278 prot = __pgprot(pgprot_val(prot) |
279 cachemode2protval(_PAGE_CACHE_MODE_WC));
280 break;
281 case _PAGE_CACHE_MODE_WT:
282 prot = __pgprot(pgprot_val(prot) |
283 cachemode2protval(_PAGE_CACHE_MODE_WT));
284 break;
285 case _PAGE_CACHE_MODE_WB:
286 break;
287 }
288
289 /*
290 * Ok, go for it..
291 */
292 area = get_vm_area_caller(size, VM_IOREMAP, caller);
293 if (!area)
294 goto err_free_memtype;
295 area->phys_addr = phys_addr;
296 vaddr = (unsigned long) area->addr;
297
298 if (memtype_kernel_map_sync(phys_addr, size, pcm))
299 goto err_free_area;
300
301 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
302 goto err_free_area;
303
304 ret_addr = (void __iomem *) (vaddr + offset);
305 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
306
307 /*
308 * Check if the request spans more than any BAR in the iomem resource
309 * tree.
310 */
311 if (iomem_map_sanity_check(unaligned_phys_addr, unaligned_size))
312 pr_warn("caller %pS mapping multiple BARs\n", caller);
313
314 return ret_addr;
315err_free_area:
316 free_vm_area(area);
317err_free_memtype:
318 memtype_free(phys_addr, phys_addr + size);
319 return NULL;
320}
321
322/**
323 * ioremap - map bus memory into CPU space
324 * @phys_addr: bus address of the memory
325 * @size: size of the resource to map
326 *
327 * ioremap performs a platform specific sequence of operations to
328 * make bus memory CPU accessible via the readb/readw/readl/writeb/
329 * writew/writel functions and the other mmio helpers. The returned
330 * address is not guaranteed to be usable directly as a virtual
331 * address.
332 *
333 * This version of ioremap ensures that the memory is marked uncachable
334 * on the CPU as well as honouring existing caching rules from things like
335 * the PCI bus. Note that there are other caches and buffers on many
336 * busses. In particular driver authors should read up on PCI writes
337 *
338 * It's useful if some control registers are in such an area and
339 * write combining or read caching is not desirable:
340 *
341 * Must be freed with iounmap.
342 */
343void __iomem *ioremap(resource_size_t phys_addr, unsigned long size)
344{
345 /*
346 * Ideally, this should be:
347 * pat_enabled() ? _PAGE_CACHE_MODE_UC : _PAGE_CACHE_MODE_UC_MINUS;
348 *
349 * Till we fix all X drivers to use ioremap_wc(), we will use
350 * UC MINUS. Drivers that are certain they need or can already
351 * be converted over to strong UC can use ioremap_uc().
352 */
353 enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC_MINUS;
354
355 return __ioremap_caller(phys_addr, size, pcm,
356 __builtin_return_address(0), false);
357}
358EXPORT_SYMBOL(ioremap);
359
360/**
361 * ioremap_uc - map bus memory into CPU space as strongly uncachable
362 * @phys_addr: bus address of the memory
363 * @size: size of the resource to map
364 *
365 * ioremap_uc performs a platform specific sequence of operations to
366 * make bus memory CPU accessible via the readb/readw/readl/writeb/
367 * writew/writel functions and the other mmio helpers. The returned
368 * address is not guaranteed to be usable directly as a virtual
369 * address.
370 *
371 * This version of ioremap ensures that the memory is marked with a strong
372 * preference as completely uncachable on the CPU when possible. For non-PAT
373 * systems this ends up setting page-attribute flags PCD=1, PWT=1. For PAT
374 * systems this will set the PAT entry for the pages as strong UC. This call
375 * will honor existing caching rules from things like the PCI bus. Note that
376 * there are other caches and buffers on many busses. In particular driver
377 * authors should read up on PCI writes.
378 *
379 * It's useful if some control registers are in such an area and
380 * write combining or read caching is not desirable:
381 *
382 * Must be freed with iounmap.
383 */
384void __iomem *ioremap_uc(resource_size_t phys_addr, unsigned long size)
385{
386 enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC;
387
388 return __ioremap_caller(phys_addr, size, pcm,
389 __builtin_return_address(0), false);
390}
391EXPORT_SYMBOL_GPL(ioremap_uc);
392
393/**
394 * ioremap_wc - map memory into CPU space write combined
395 * @phys_addr: bus address of the memory
396 * @size: size of the resource to map
397 *
398 * This version of ioremap ensures that the memory is marked write combining.
399 * Write combining allows faster writes to some hardware devices.
400 *
401 * Must be freed with iounmap.
402 */
403void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
404{
405 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WC,
406 __builtin_return_address(0), false);
407}
408EXPORT_SYMBOL(ioremap_wc);
409
410/**
411 * ioremap_wt - map memory into CPU space write through
412 * @phys_addr: bus address of the memory
413 * @size: size of the resource to map
414 *
415 * This version of ioremap ensures that the memory is marked write through.
416 * Write through stores data into memory while keeping the cache up-to-date.
417 *
418 * Must be freed with iounmap.
419 */
420void __iomem *ioremap_wt(resource_size_t phys_addr, unsigned long size)
421{
422 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WT,
423 __builtin_return_address(0), false);
424}
425EXPORT_SYMBOL(ioremap_wt);
426
427void __iomem *ioremap_encrypted(resource_size_t phys_addr, unsigned long size)
428{
429 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
430 __builtin_return_address(0), true);
431}
432EXPORT_SYMBOL(ioremap_encrypted);
433
434void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
435{
436 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
437 __builtin_return_address(0), false);
438}
439EXPORT_SYMBOL(ioremap_cache);
440
441void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
442 unsigned long prot_val)
443{
444 return __ioremap_caller(phys_addr, size,
445 pgprot2cachemode(__pgprot(prot_val)),
446 __builtin_return_address(0), false);
447}
448EXPORT_SYMBOL(ioremap_prot);
449
450/**
451 * iounmap - Free a IO remapping
452 * @addr: virtual address from ioremap_*
453 *
454 * Caller must ensure there is only one unmapping for the same pointer.
455 */
456void iounmap(volatile void __iomem *addr)
457{
458 struct vm_struct *p, *o;
459
460 if ((void __force *)addr <= high_memory)
461 return;
462
463 /*
464 * The PCI/ISA range special-casing was removed from __ioremap()
465 * so this check, in theory, can be removed. However, there are
466 * cases where iounmap() is called for addresses not obtained via
467 * ioremap() (vga16fb for example). Add a warning so that these
468 * cases can be caught and fixed.
469 */
470 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
471 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS)) {
472 WARN(1, "iounmap() called for ISA range not obtained using ioremap()\n");
473 return;
474 }
475
476 mmiotrace_iounmap(addr);
477
478 addr = (volatile void __iomem *)
479 (PAGE_MASK & (unsigned long __force)addr);
480
481 /* Use the vm area unlocked, assuming the caller
482 ensures there isn't another iounmap for the same address
483 in parallel. Reuse of the virtual address is prevented by
484 leaving it in the global lists until we're done with it.
485 cpa takes care of the direct mappings. */
486 p = find_vm_area((void __force *)addr);
487
488 if (!p) {
489 printk(KERN_ERR "iounmap: bad address %p\n", addr);
490 dump_stack();
491 return;
492 }
493
494 kmsan_iounmap_page_range((unsigned long)addr,
495 (unsigned long)addr + get_vm_area_size(p));
496 memtype_free(p->phys_addr, p->phys_addr + get_vm_area_size(p));
497
498 /* Finally remove it */
499 o = remove_vm_area((void __force *)addr);
500 BUG_ON(p != o || o == NULL);
501 kfree(p);
502}
503EXPORT_SYMBOL(iounmap);
504
505/*
506 * Convert a physical pointer to a virtual kernel pointer for /dev/mem
507 * access
508 */
509void *xlate_dev_mem_ptr(phys_addr_t phys)
510{
511 unsigned long start = phys & PAGE_MASK;
512 unsigned long offset = phys & ~PAGE_MASK;
513 void *vaddr;
514
515 /* memremap() maps if RAM, otherwise falls back to ioremap() */
516 vaddr = memremap(start, PAGE_SIZE, MEMREMAP_WB);
517
518 /* Only add the offset on success and return NULL if memremap() failed */
519 if (vaddr)
520 vaddr += offset;
521
522 return vaddr;
523}
524
525void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
526{
527 memunmap((void *)((unsigned long)addr & PAGE_MASK));
528}
529
530#ifdef CONFIG_AMD_MEM_ENCRYPT
531/*
532 * Examine the physical address to determine if it is an area of memory
533 * that should be mapped decrypted. If the memory is not part of the
534 * kernel usable area it was accessed and created decrypted, so these
535 * areas should be mapped decrypted. And since the encryption key can
536 * change across reboots, persistent memory should also be mapped
537 * decrypted.
538 *
539 * If SEV is active, that implies that BIOS/UEFI also ran encrypted so
540 * only persistent memory should be mapped decrypted.
541 */
542static bool memremap_should_map_decrypted(resource_size_t phys_addr,
543 unsigned long size)
544{
545 int is_pmem;
546
547 /*
548 * Check if the address is part of a persistent memory region.
549 * This check covers areas added by E820, EFI and ACPI.
550 */
551 is_pmem = region_intersects(phys_addr, size, IORESOURCE_MEM,
552 IORES_DESC_PERSISTENT_MEMORY);
553 if (is_pmem != REGION_DISJOINT)
554 return true;
555
556 /*
557 * Check if the non-volatile attribute is set for an EFI
558 * reserved area.
559 */
560 if (efi_enabled(EFI_BOOT)) {
561 switch (efi_mem_type(phys_addr)) {
562 case EFI_RESERVED_TYPE:
563 if (efi_mem_attributes(phys_addr) & EFI_MEMORY_NV)
564 return true;
565 break;
566 default:
567 break;
568 }
569 }
570
571 /* Check if the address is outside kernel usable area */
572 switch (e820__get_entry_type(phys_addr, phys_addr + size - 1)) {
573 case E820_TYPE_RESERVED:
574 case E820_TYPE_ACPI:
575 case E820_TYPE_NVS:
576 case E820_TYPE_UNUSABLE:
577 /* For SEV, these areas are encrypted */
578 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
579 break;
580 fallthrough;
581
582 case E820_TYPE_PRAM:
583 return true;
584 default:
585 break;
586 }
587
588 return false;
589}
590
591/*
592 * Examine the physical address to determine if it is EFI data. Check
593 * it against the boot params structure and EFI tables and memory types.
594 */
595static bool memremap_is_efi_data(resource_size_t phys_addr,
596 unsigned long size)
597{
598 u64 paddr;
599
600 /* Check if the address is part of EFI boot/runtime data */
601 if (!efi_enabled(EFI_BOOT))
602 return false;
603
604 paddr = boot_params.efi_info.efi_memmap_hi;
605 paddr <<= 32;
606 paddr |= boot_params.efi_info.efi_memmap;
607 if (phys_addr == paddr)
608 return true;
609
610 paddr = boot_params.efi_info.efi_systab_hi;
611 paddr <<= 32;
612 paddr |= boot_params.efi_info.efi_systab;
613 if (phys_addr == paddr)
614 return true;
615
616 if (efi_is_table_address(phys_addr))
617 return true;
618
619 switch (efi_mem_type(phys_addr)) {
620 case EFI_BOOT_SERVICES_DATA:
621 case EFI_RUNTIME_SERVICES_DATA:
622 return true;
623 default:
624 break;
625 }
626
627 return false;
628}
629
630/*
631 * Examine the physical address to determine if it is boot data by checking
632 * it against the boot params setup_data chain.
633 */
634static bool memremap_is_setup_data(resource_size_t phys_addr,
635 unsigned long size)
636{
637 struct setup_indirect *indirect;
638 struct setup_data *data;
639 u64 paddr, paddr_next;
640
641 paddr = boot_params.hdr.setup_data;
642 while (paddr) {
643 unsigned int len;
644
645 if (phys_addr == paddr)
646 return true;
647
648 data = memremap(paddr, sizeof(*data),
649 MEMREMAP_WB | MEMREMAP_DEC);
650 if (!data) {
651 pr_warn("failed to memremap setup_data entry\n");
652 return false;
653 }
654
655 paddr_next = data->next;
656 len = data->len;
657
658 if ((phys_addr > paddr) && (phys_addr < (paddr + len))) {
659 memunmap(data);
660 return true;
661 }
662
663 if (data->type == SETUP_INDIRECT) {
664 memunmap(data);
665 data = memremap(paddr, sizeof(*data) + len,
666 MEMREMAP_WB | MEMREMAP_DEC);
667 if (!data) {
668 pr_warn("failed to memremap indirect setup_data\n");
669 return false;
670 }
671
672 indirect = (struct setup_indirect *)data->data;
673
674 if (indirect->type != SETUP_INDIRECT) {
675 paddr = indirect->addr;
676 len = indirect->len;
677 }
678 }
679
680 memunmap(data);
681
682 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
683 return true;
684
685 paddr = paddr_next;
686 }
687
688 return false;
689}
690
691/*
692 * Examine the physical address to determine if it is boot data by checking
693 * it against the boot params setup_data chain (early boot version).
694 */
695static bool __init early_memremap_is_setup_data(resource_size_t phys_addr,
696 unsigned long size)
697{
698 struct setup_indirect *indirect;
699 struct setup_data *data;
700 u64 paddr, paddr_next;
701
702 paddr = boot_params.hdr.setup_data;
703 while (paddr) {
704 unsigned int len, size;
705
706 if (phys_addr == paddr)
707 return true;
708
709 data = early_memremap_decrypted(paddr, sizeof(*data));
710 if (!data) {
711 pr_warn("failed to early memremap setup_data entry\n");
712 return false;
713 }
714
715 size = sizeof(*data);
716
717 paddr_next = data->next;
718 len = data->len;
719
720 if ((phys_addr > paddr) && (phys_addr < (paddr + len))) {
721 early_memunmap(data, sizeof(*data));
722 return true;
723 }
724
725 if (data->type == SETUP_INDIRECT) {
726 size += len;
727 early_memunmap(data, sizeof(*data));
728 data = early_memremap_decrypted(paddr, size);
729 if (!data) {
730 pr_warn("failed to early memremap indirect setup_data\n");
731 return false;
732 }
733
734 indirect = (struct setup_indirect *)data->data;
735
736 if (indirect->type != SETUP_INDIRECT) {
737 paddr = indirect->addr;
738 len = indirect->len;
739 }
740 }
741
742 early_memunmap(data, size);
743
744 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
745 return true;
746
747 paddr = paddr_next;
748 }
749
750 return false;
751}
752
753/*
754 * Architecture function to determine if RAM remap is allowed. By default, a
755 * RAM remap will map the data as encrypted. Determine if a RAM remap should
756 * not be done so that the data will be mapped decrypted.
757 */
758bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size,
759 unsigned long flags)
760{
761 if (!cc_platform_has(CC_ATTR_MEM_ENCRYPT))
762 return true;
763
764 if (flags & MEMREMAP_ENC)
765 return true;
766
767 if (flags & MEMREMAP_DEC)
768 return false;
769
770 if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
771 if (memremap_is_setup_data(phys_addr, size) ||
772 memremap_is_efi_data(phys_addr, size))
773 return false;
774 }
775
776 return !memremap_should_map_decrypted(phys_addr, size);
777}
778
779/*
780 * Architecture override of __weak function to adjust the protection attributes
781 * used when remapping memory. By default, early_memremap() will map the data
782 * as encrypted. Determine if an encrypted mapping should not be done and set
783 * the appropriate protection attributes.
784 */
785pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr,
786 unsigned long size,
787 pgprot_t prot)
788{
789 bool encrypted_prot;
790
791 if (!cc_platform_has(CC_ATTR_MEM_ENCRYPT))
792 return prot;
793
794 encrypted_prot = true;
795
796 if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
797 if (early_memremap_is_setup_data(phys_addr, size) ||
798 memremap_is_efi_data(phys_addr, size))
799 encrypted_prot = false;
800 }
801
802 if (encrypted_prot && memremap_should_map_decrypted(phys_addr, size))
803 encrypted_prot = false;
804
805 return encrypted_prot ? pgprot_encrypted(prot)
806 : pgprot_decrypted(prot);
807}
808
809bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size)
810{
811 return arch_memremap_can_ram_remap(phys_addr, size, 0);
812}
813
814/* Remap memory with encryption */
815void __init *early_memremap_encrypted(resource_size_t phys_addr,
816 unsigned long size)
817{
818 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC);
819}
820
821/*
822 * Remap memory with encryption and write-protected - cannot be called
823 * before pat_init() is called
824 */
825void __init *early_memremap_encrypted_wp(resource_size_t phys_addr,
826 unsigned long size)
827{
828 if (!x86_has_pat_wp())
829 return NULL;
830 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC_WP);
831}
832
833/* Remap memory without encryption */
834void __init *early_memremap_decrypted(resource_size_t phys_addr,
835 unsigned long size)
836{
837 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC);
838}
839
840/*
841 * Remap memory without encryption and write-protected - cannot be called
842 * before pat_init() is called
843 */
844void __init *early_memremap_decrypted_wp(resource_size_t phys_addr,
845 unsigned long size)
846{
847 if (!x86_has_pat_wp())
848 return NULL;
849 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC_WP);
850}
851#endif /* CONFIG_AMD_MEM_ENCRYPT */
852
853static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
854
855static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
856{
857 /* Don't assume we're using swapper_pg_dir at this point */
858 pgd_t *base = __va(read_cr3_pa());
859 pgd_t *pgd = &base[pgd_index(addr)];
860 p4d_t *p4d = p4d_offset(pgd, addr);
861 pud_t *pud = pud_offset(p4d, addr);
862 pmd_t *pmd = pmd_offset(pud, addr);
863
864 return pmd;
865}
866
867static inline pte_t * __init early_ioremap_pte(unsigned long addr)
868{
869 return &bm_pte[pte_index(addr)];
870}
871
872bool __init is_early_ioremap_ptep(pte_t *ptep)
873{
874 return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
875}
876
877void __init early_ioremap_init(void)
878{
879 pmd_t *pmd;
880
881#ifdef CONFIG_X86_64
882 BUILD_BUG_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
883#else
884 WARN_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
885#endif
886
887 early_ioremap_setup();
888
889 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
890 memset(bm_pte, 0, sizeof(bm_pte));
891 pmd_populate_kernel(&init_mm, pmd, bm_pte);
892
893 /*
894 * The boot-ioremap range spans multiple pmds, for which
895 * we are not prepared:
896 */
897#define __FIXADDR_TOP (-PAGE_SIZE)
898 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
899 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
900#undef __FIXADDR_TOP
901 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
902 WARN_ON(1);
903 printk(KERN_WARNING "pmd %p != %p\n",
904 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
905 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
906 fix_to_virt(FIX_BTMAP_BEGIN));
907 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
908 fix_to_virt(FIX_BTMAP_END));
909
910 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
911 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
912 FIX_BTMAP_BEGIN);
913 }
914}
915
916void __init __early_set_fixmap(enum fixed_addresses idx,
917 phys_addr_t phys, pgprot_t flags)
918{
919 unsigned long addr = __fix_to_virt(idx);
920 pte_t *pte;
921
922 if (idx >= __end_of_fixed_addresses) {
923 BUG();
924 return;
925 }
926 pte = early_ioremap_pte(addr);
927
928 /* Sanitize 'prot' against any unsupported bits: */
929 pgprot_val(flags) &= __supported_pte_mask;
930
931 if (pgprot_val(flags))
932 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
933 else
934 pte_clear(&init_mm, addr, pte);
935 flush_tlb_one_kernel(addr);
936}
1/*
2 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
5 *
6 * (C) Copyright 1995 1996 Linus Torvalds
7 */
8
9#include <linux/bootmem.h>
10#include <linux/init.h>
11#include <linux/io.h>
12#include <linux/ioport.h>
13#include <linux/slab.h>
14#include <linux/vmalloc.h>
15#include <linux/mmiotrace.h>
16#include <linux/mem_encrypt.h>
17#include <linux/efi.h>
18
19#include <asm/set_memory.h>
20#include <asm/e820/api.h>
21#include <asm/fixmap.h>
22#include <asm/pgtable.h>
23#include <asm/tlbflush.h>
24#include <asm/pgalloc.h>
25#include <asm/pat.h>
26#include <asm/setup.h>
27
28#include "physaddr.h"
29
30struct ioremap_mem_flags {
31 bool system_ram;
32 bool desc_other;
33};
34
35/*
36 * Fix up the linear direct mapping of the kernel to avoid cache attribute
37 * conflicts.
38 */
39int ioremap_change_attr(unsigned long vaddr, unsigned long size,
40 enum page_cache_mode pcm)
41{
42 unsigned long nrpages = size >> PAGE_SHIFT;
43 int err;
44
45 switch (pcm) {
46 case _PAGE_CACHE_MODE_UC:
47 default:
48 err = _set_memory_uc(vaddr, nrpages);
49 break;
50 case _PAGE_CACHE_MODE_WC:
51 err = _set_memory_wc(vaddr, nrpages);
52 break;
53 case _PAGE_CACHE_MODE_WT:
54 err = _set_memory_wt(vaddr, nrpages);
55 break;
56 case _PAGE_CACHE_MODE_WB:
57 err = _set_memory_wb(vaddr, nrpages);
58 break;
59 }
60
61 return err;
62}
63
64static bool __ioremap_check_ram(struct resource *res)
65{
66 unsigned long start_pfn, stop_pfn;
67 unsigned long i;
68
69 if ((res->flags & IORESOURCE_SYSTEM_RAM) != IORESOURCE_SYSTEM_RAM)
70 return false;
71
72 start_pfn = (res->start + PAGE_SIZE - 1) >> PAGE_SHIFT;
73 stop_pfn = (res->end + 1) >> PAGE_SHIFT;
74 if (stop_pfn > start_pfn) {
75 for (i = 0; i < (stop_pfn - start_pfn); ++i)
76 if (pfn_valid(start_pfn + i) &&
77 !PageReserved(pfn_to_page(start_pfn + i)))
78 return true;
79 }
80
81 return false;
82}
83
84static int __ioremap_check_desc_other(struct resource *res)
85{
86 return (res->desc != IORES_DESC_NONE);
87}
88
89static int __ioremap_res_check(struct resource *res, void *arg)
90{
91 struct ioremap_mem_flags *flags = arg;
92
93 if (!flags->system_ram)
94 flags->system_ram = __ioremap_check_ram(res);
95
96 if (!flags->desc_other)
97 flags->desc_other = __ioremap_check_desc_other(res);
98
99 return flags->system_ram && flags->desc_other;
100}
101
102/*
103 * To avoid multiple resource walks, this function walks resources marked as
104 * IORESOURCE_MEM and IORESOURCE_BUSY and looking for system RAM and/or a
105 * resource described not as IORES_DESC_NONE (e.g. IORES_DESC_ACPI_TABLES).
106 */
107static void __ioremap_check_mem(resource_size_t addr, unsigned long size,
108 struct ioremap_mem_flags *flags)
109{
110 u64 start, end;
111
112 start = (u64)addr;
113 end = start + size - 1;
114 memset(flags, 0, sizeof(*flags));
115
116 walk_mem_res(start, end, flags, __ioremap_res_check);
117}
118
119/*
120 * Remap an arbitrary physical address space into the kernel virtual
121 * address space. It transparently creates kernel huge I/O mapping when
122 * the physical address is aligned by a huge page size (1GB or 2MB) and
123 * the requested size is at least the huge page size.
124 *
125 * NOTE: MTRRs can override PAT memory types with a 4KB granularity.
126 * Therefore, the mapping code falls back to use a smaller page toward 4KB
127 * when a mapping range is covered by non-WB type of MTRRs.
128 *
129 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
130 * have to convert them into an offset in a page-aligned mapping, but the
131 * caller shouldn't need to know that small detail.
132 */
133static void __iomem *__ioremap_caller(resource_size_t phys_addr,
134 unsigned long size, enum page_cache_mode pcm, void *caller)
135{
136 unsigned long offset, vaddr;
137 resource_size_t last_addr;
138 const resource_size_t unaligned_phys_addr = phys_addr;
139 const unsigned long unaligned_size = size;
140 struct ioremap_mem_flags mem_flags;
141 struct vm_struct *area;
142 enum page_cache_mode new_pcm;
143 pgprot_t prot;
144 int retval;
145 void __iomem *ret_addr;
146
147 /* Don't allow wraparound or zero size */
148 last_addr = phys_addr + size - 1;
149 if (!size || last_addr < phys_addr)
150 return NULL;
151
152 if (!phys_addr_valid(phys_addr)) {
153 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
154 (unsigned long long)phys_addr);
155 WARN_ON_ONCE(1);
156 return NULL;
157 }
158
159 __ioremap_check_mem(phys_addr, size, &mem_flags);
160
161 /*
162 * Don't allow anybody to remap normal RAM that we're using..
163 */
164 if (mem_flags.system_ram) {
165 WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n",
166 &phys_addr, &last_addr);
167 return NULL;
168 }
169
170 /*
171 * Mappings have to be page-aligned
172 */
173 offset = phys_addr & ~PAGE_MASK;
174 phys_addr &= PHYSICAL_PAGE_MASK;
175 size = PAGE_ALIGN(last_addr+1) - phys_addr;
176
177 retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
178 pcm, &new_pcm);
179 if (retval) {
180 printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
181 return NULL;
182 }
183
184 if (pcm != new_pcm) {
185 if (!is_new_memtype_allowed(phys_addr, size, pcm, new_pcm)) {
186 printk(KERN_ERR
187 "ioremap error for 0x%llx-0x%llx, requested 0x%x, got 0x%x\n",
188 (unsigned long long)phys_addr,
189 (unsigned long long)(phys_addr + size),
190 pcm, new_pcm);
191 goto err_free_memtype;
192 }
193 pcm = new_pcm;
194 }
195
196 /*
197 * If the page being mapped is in memory and SEV is active then
198 * make sure the memory encryption attribute is enabled in the
199 * resulting mapping.
200 */
201 prot = PAGE_KERNEL_IO;
202 if (sev_active() && mem_flags.desc_other)
203 prot = pgprot_encrypted(prot);
204
205 switch (pcm) {
206 case _PAGE_CACHE_MODE_UC:
207 default:
208 prot = __pgprot(pgprot_val(prot) |
209 cachemode2protval(_PAGE_CACHE_MODE_UC));
210 break;
211 case _PAGE_CACHE_MODE_UC_MINUS:
212 prot = __pgprot(pgprot_val(prot) |
213 cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS));
214 break;
215 case _PAGE_CACHE_MODE_WC:
216 prot = __pgprot(pgprot_val(prot) |
217 cachemode2protval(_PAGE_CACHE_MODE_WC));
218 break;
219 case _PAGE_CACHE_MODE_WT:
220 prot = __pgprot(pgprot_val(prot) |
221 cachemode2protval(_PAGE_CACHE_MODE_WT));
222 break;
223 case _PAGE_CACHE_MODE_WB:
224 break;
225 }
226
227 /*
228 * Ok, go for it..
229 */
230 area = get_vm_area_caller(size, VM_IOREMAP, caller);
231 if (!area)
232 goto err_free_memtype;
233 area->phys_addr = phys_addr;
234 vaddr = (unsigned long) area->addr;
235
236 if (kernel_map_sync_memtype(phys_addr, size, pcm))
237 goto err_free_area;
238
239 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
240 goto err_free_area;
241
242 ret_addr = (void __iomem *) (vaddr + offset);
243 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
244
245 /*
246 * Check if the request spans more than any BAR in the iomem resource
247 * tree.
248 */
249 if (iomem_map_sanity_check(unaligned_phys_addr, unaligned_size))
250 pr_warn("caller %pS mapping multiple BARs\n", caller);
251
252 return ret_addr;
253err_free_area:
254 free_vm_area(area);
255err_free_memtype:
256 free_memtype(phys_addr, phys_addr + size);
257 return NULL;
258}
259
260/**
261 * ioremap_nocache - map bus memory into CPU space
262 * @phys_addr: bus address of the memory
263 * @size: size of the resource to map
264 *
265 * ioremap_nocache performs a platform specific sequence of operations to
266 * make bus memory CPU accessible via the readb/readw/readl/writeb/
267 * writew/writel functions and the other mmio helpers. The returned
268 * address is not guaranteed to be usable directly as a virtual
269 * address.
270 *
271 * This version of ioremap ensures that the memory is marked uncachable
272 * on the CPU as well as honouring existing caching rules from things like
273 * the PCI bus. Note that there are other caches and buffers on many
274 * busses. In particular driver authors should read up on PCI writes
275 *
276 * It's useful if some control registers are in such an area and
277 * write combining or read caching is not desirable:
278 *
279 * Must be freed with iounmap.
280 */
281void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
282{
283 /*
284 * Ideally, this should be:
285 * pat_enabled() ? _PAGE_CACHE_MODE_UC : _PAGE_CACHE_MODE_UC_MINUS;
286 *
287 * Till we fix all X drivers to use ioremap_wc(), we will use
288 * UC MINUS. Drivers that are certain they need or can already
289 * be converted over to strong UC can use ioremap_uc().
290 */
291 enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC_MINUS;
292
293 return __ioremap_caller(phys_addr, size, pcm,
294 __builtin_return_address(0));
295}
296EXPORT_SYMBOL(ioremap_nocache);
297
298/**
299 * ioremap_uc - map bus memory into CPU space as strongly uncachable
300 * @phys_addr: bus address of the memory
301 * @size: size of the resource to map
302 *
303 * ioremap_uc performs a platform specific sequence of operations to
304 * make bus memory CPU accessible via the readb/readw/readl/writeb/
305 * writew/writel functions and the other mmio helpers. The returned
306 * address is not guaranteed to be usable directly as a virtual
307 * address.
308 *
309 * This version of ioremap ensures that the memory is marked with a strong
310 * preference as completely uncachable on the CPU when possible. For non-PAT
311 * systems this ends up setting page-attribute flags PCD=1, PWT=1. For PAT
312 * systems this will set the PAT entry for the pages as strong UC. This call
313 * will honor existing caching rules from things like the PCI bus. Note that
314 * there are other caches and buffers on many busses. In particular driver
315 * authors should read up on PCI writes.
316 *
317 * It's useful if some control registers are in such an area and
318 * write combining or read caching is not desirable:
319 *
320 * Must be freed with iounmap.
321 */
322void __iomem *ioremap_uc(resource_size_t phys_addr, unsigned long size)
323{
324 enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC;
325
326 return __ioremap_caller(phys_addr, size, pcm,
327 __builtin_return_address(0));
328}
329EXPORT_SYMBOL_GPL(ioremap_uc);
330
331/**
332 * ioremap_wc - map memory into CPU space write combined
333 * @phys_addr: bus address of the memory
334 * @size: size of the resource to map
335 *
336 * This version of ioremap ensures that the memory is marked write combining.
337 * Write combining allows faster writes to some hardware devices.
338 *
339 * Must be freed with iounmap.
340 */
341void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
342{
343 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WC,
344 __builtin_return_address(0));
345}
346EXPORT_SYMBOL(ioremap_wc);
347
348/**
349 * ioremap_wt - map memory into CPU space write through
350 * @phys_addr: bus address of the memory
351 * @size: size of the resource to map
352 *
353 * This version of ioremap ensures that the memory is marked write through.
354 * Write through stores data into memory while keeping the cache up-to-date.
355 *
356 * Must be freed with iounmap.
357 */
358void __iomem *ioremap_wt(resource_size_t phys_addr, unsigned long size)
359{
360 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WT,
361 __builtin_return_address(0));
362}
363EXPORT_SYMBOL(ioremap_wt);
364
365void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
366{
367 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
368 __builtin_return_address(0));
369}
370EXPORT_SYMBOL(ioremap_cache);
371
372void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
373 unsigned long prot_val)
374{
375 return __ioremap_caller(phys_addr, size,
376 pgprot2cachemode(__pgprot(prot_val)),
377 __builtin_return_address(0));
378}
379EXPORT_SYMBOL(ioremap_prot);
380
381/**
382 * iounmap - Free a IO remapping
383 * @addr: virtual address from ioremap_*
384 *
385 * Caller must ensure there is only one unmapping for the same pointer.
386 */
387void iounmap(volatile void __iomem *addr)
388{
389 struct vm_struct *p, *o;
390
391 if ((void __force *)addr <= high_memory)
392 return;
393
394 /*
395 * The PCI/ISA range special-casing was removed from __ioremap()
396 * so this check, in theory, can be removed. However, there are
397 * cases where iounmap() is called for addresses not obtained via
398 * ioremap() (vga16fb for example). Add a warning so that these
399 * cases can be caught and fixed.
400 */
401 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
402 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS)) {
403 WARN(1, "iounmap() called for ISA range not obtained using ioremap()\n");
404 return;
405 }
406
407 mmiotrace_iounmap(addr);
408
409 addr = (volatile void __iomem *)
410 (PAGE_MASK & (unsigned long __force)addr);
411
412 /* Use the vm area unlocked, assuming the caller
413 ensures there isn't another iounmap for the same address
414 in parallel. Reuse of the virtual address is prevented by
415 leaving it in the global lists until we're done with it.
416 cpa takes care of the direct mappings. */
417 p = find_vm_area((void __force *)addr);
418
419 if (!p) {
420 printk(KERN_ERR "iounmap: bad address %p\n", addr);
421 dump_stack();
422 return;
423 }
424
425 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
426
427 /* Finally remove it */
428 o = remove_vm_area((void __force *)addr);
429 BUG_ON(p != o || o == NULL);
430 kfree(p);
431}
432EXPORT_SYMBOL(iounmap);
433
434int __init arch_ioremap_pud_supported(void)
435{
436#ifdef CONFIG_X86_64
437 return boot_cpu_has(X86_FEATURE_GBPAGES);
438#else
439 return 0;
440#endif
441}
442
443int __init arch_ioremap_pmd_supported(void)
444{
445 return boot_cpu_has(X86_FEATURE_PSE);
446}
447
448/*
449 * Convert a physical pointer to a virtual kernel pointer for /dev/mem
450 * access
451 */
452void *xlate_dev_mem_ptr(phys_addr_t phys)
453{
454 unsigned long start = phys & PAGE_MASK;
455 unsigned long offset = phys & ~PAGE_MASK;
456 void *vaddr;
457
458 /* memremap() maps if RAM, otherwise falls back to ioremap() */
459 vaddr = memremap(start, PAGE_SIZE, MEMREMAP_WB);
460
461 /* Only add the offset on success and return NULL if memremap() failed */
462 if (vaddr)
463 vaddr += offset;
464
465 return vaddr;
466}
467
468void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
469{
470 memunmap((void *)((unsigned long)addr & PAGE_MASK));
471}
472
473/*
474 * Examine the physical address to determine if it is an area of memory
475 * that should be mapped decrypted. If the memory is not part of the
476 * kernel usable area it was accessed and created decrypted, so these
477 * areas should be mapped decrypted. And since the encryption key can
478 * change across reboots, persistent memory should also be mapped
479 * decrypted.
480 *
481 * If SEV is active, that implies that BIOS/UEFI also ran encrypted so
482 * only persistent memory should be mapped decrypted.
483 */
484static bool memremap_should_map_decrypted(resource_size_t phys_addr,
485 unsigned long size)
486{
487 int is_pmem;
488
489 /*
490 * Check if the address is part of a persistent memory region.
491 * This check covers areas added by E820, EFI and ACPI.
492 */
493 is_pmem = region_intersects(phys_addr, size, IORESOURCE_MEM,
494 IORES_DESC_PERSISTENT_MEMORY);
495 if (is_pmem != REGION_DISJOINT)
496 return true;
497
498 /*
499 * Check if the non-volatile attribute is set for an EFI
500 * reserved area.
501 */
502 if (efi_enabled(EFI_BOOT)) {
503 switch (efi_mem_type(phys_addr)) {
504 case EFI_RESERVED_TYPE:
505 if (efi_mem_attributes(phys_addr) & EFI_MEMORY_NV)
506 return true;
507 break;
508 default:
509 break;
510 }
511 }
512
513 /* Check if the address is outside kernel usable area */
514 switch (e820__get_entry_type(phys_addr, phys_addr + size - 1)) {
515 case E820_TYPE_RESERVED:
516 case E820_TYPE_ACPI:
517 case E820_TYPE_NVS:
518 case E820_TYPE_UNUSABLE:
519 /* For SEV, these areas are encrypted */
520 if (sev_active())
521 break;
522 /* Fallthrough */
523
524 case E820_TYPE_PRAM:
525 return true;
526 default:
527 break;
528 }
529
530 return false;
531}
532
533/*
534 * Examine the physical address to determine if it is EFI data. Check
535 * it against the boot params structure and EFI tables and memory types.
536 */
537static bool memremap_is_efi_data(resource_size_t phys_addr,
538 unsigned long size)
539{
540 u64 paddr;
541
542 /* Check if the address is part of EFI boot/runtime data */
543 if (!efi_enabled(EFI_BOOT))
544 return false;
545
546 paddr = boot_params.efi_info.efi_memmap_hi;
547 paddr <<= 32;
548 paddr |= boot_params.efi_info.efi_memmap;
549 if (phys_addr == paddr)
550 return true;
551
552 paddr = boot_params.efi_info.efi_systab_hi;
553 paddr <<= 32;
554 paddr |= boot_params.efi_info.efi_systab;
555 if (phys_addr == paddr)
556 return true;
557
558 if (efi_is_table_address(phys_addr))
559 return true;
560
561 switch (efi_mem_type(phys_addr)) {
562 case EFI_BOOT_SERVICES_DATA:
563 case EFI_RUNTIME_SERVICES_DATA:
564 return true;
565 default:
566 break;
567 }
568
569 return false;
570}
571
572/*
573 * Examine the physical address to determine if it is boot data by checking
574 * it against the boot params setup_data chain.
575 */
576static bool memremap_is_setup_data(resource_size_t phys_addr,
577 unsigned long size)
578{
579 struct setup_data *data;
580 u64 paddr, paddr_next;
581
582 paddr = boot_params.hdr.setup_data;
583 while (paddr) {
584 unsigned int len;
585
586 if (phys_addr == paddr)
587 return true;
588
589 data = memremap(paddr, sizeof(*data),
590 MEMREMAP_WB | MEMREMAP_DEC);
591
592 paddr_next = data->next;
593 len = data->len;
594
595 memunmap(data);
596
597 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
598 return true;
599
600 paddr = paddr_next;
601 }
602
603 return false;
604}
605
606/*
607 * Examine the physical address to determine if it is boot data by checking
608 * it against the boot params setup_data chain (early boot version).
609 */
610static bool __init early_memremap_is_setup_data(resource_size_t phys_addr,
611 unsigned long size)
612{
613 struct setup_data *data;
614 u64 paddr, paddr_next;
615
616 paddr = boot_params.hdr.setup_data;
617 while (paddr) {
618 unsigned int len;
619
620 if (phys_addr == paddr)
621 return true;
622
623 data = early_memremap_decrypted(paddr, sizeof(*data));
624
625 paddr_next = data->next;
626 len = data->len;
627
628 early_memunmap(data, sizeof(*data));
629
630 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
631 return true;
632
633 paddr = paddr_next;
634 }
635
636 return false;
637}
638
639/*
640 * Architecture function to determine if RAM remap is allowed. By default, a
641 * RAM remap will map the data as encrypted. Determine if a RAM remap should
642 * not be done so that the data will be mapped decrypted.
643 */
644bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size,
645 unsigned long flags)
646{
647 if (!mem_encrypt_active())
648 return true;
649
650 if (flags & MEMREMAP_ENC)
651 return true;
652
653 if (flags & MEMREMAP_DEC)
654 return false;
655
656 if (sme_active()) {
657 if (memremap_is_setup_data(phys_addr, size) ||
658 memremap_is_efi_data(phys_addr, size))
659 return false;
660 }
661
662 return !memremap_should_map_decrypted(phys_addr, size);
663}
664
665/*
666 * Architecture override of __weak function to adjust the protection attributes
667 * used when remapping memory. By default, early_memremap() will map the data
668 * as encrypted. Determine if an encrypted mapping should not be done and set
669 * the appropriate protection attributes.
670 */
671pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr,
672 unsigned long size,
673 pgprot_t prot)
674{
675 bool encrypted_prot;
676
677 if (!mem_encrypt_active())
678 return prot;
679
680 encrypted_prot = true;
681
682 if (sme_active()) {
683 if (early_memremap_is_setup_data(phys_addr, size) ||
684 memremap_is_efi_data(phys_addr, size))
685 encrypted_prot = false;
686 }
687
688 if (encrypted_prot && memremap_should_map_decrypted(phys_addr, size))
689 encrypted_prot = false;
690
691 return encrypted_prot ? pgprot_encrypted(prot)
692 : pgprot_decrypted(prot);
693}
694
695bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size)
696{
697 return arch_memremap_can_ram_remap(phys_addr, size, 0);
698}
699
700#ifdef CONFIG_ARCH_USE_MEMREMAP_PROT
701/* Remap memory with encryption */
702void __init *early_memremap_encrypted(resource_size_t phys_addr,
703 unsigned long size)
704{
705 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC);
706}
707
708/*
709 * Remap memory with encryption and write-protected - cannot be called
710 * before pat_init() is called
711 */
712void __init *early_memremap_encrypted_wp(resource_size_t phys_addr,
713 unsigned long size)
714{
715 /* Be sure the write-protect PAT entry is set for write-protect */
716 if (__pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] != _PAGE_CACHE_MODE_WP)
717 return NULL;
718
719 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC_WP);
720}
721
722/* Remap memory without encryption */
723void __init *early_memremap_decrypted(resource_size_t phys_addr,
724 unsigned long size)
725{
726 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC);
727}
728
729/*
730 * Remap memory without encryption and write-protected - cannot be called
731 * before pat_init() is called
732 */
733void __init *early_memremap_decrypted_wp(resource_size_t phys_addr,
734 unsigned long size)
735{
736 /* Be sure the write-protect PAT entry is set for write-protect */
737 if (__pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] != _PAGE_CACHE_MODE_WP)
738 return NULL;
739
740 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC_WP);
741}
742#endif /* CONFIG_ARCH_USE_MEMREMAP_PROT */
743
744static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
745
746static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
747{
748 /* Don't assume we're using swapper_pg_dir at this point */
749 pgd_t *base = __va(read_cr3_pa());
750 pgd_t *pgd = &base[pgd_index(addr)];
751 p4d_t *p4d = p4d_offset(pgd, addr);
752 pud_t *pud = pud_offset(p4d, addr);
753 pmd_t *pmd = pmd_offset(pud, addr);
754
755 return pmd;
756}
757
758static inline pte_t * __init early_ioremap_pte(unsigned long addr)
759{
760 return &bm_pte[pte_index(addr)];
761}
762
763bool __init is_early_ioremap_ptep(pte_t *ptep)
764{
765 return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
766}
767
768void __init early_ioremap_init(void)
769{
770 pmd_t *pmd;
771
772#ifdef CONFIG_X86_64
773 BUILD_BUG_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
774#else
775 WARN_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
776#endif
777
778 early_ioremap_setup();
779
780 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
781 memset(bm_pte, 0, sizeof(bm_pte));
782 pmd_populate_kernel(&init_mm, pmd, bm_pte);
783
784 /*
785 * The boot-ioremap range spans multiple pmds, for which
786 * we are not prepared:
787 */
788#define __FIXADDR_TOP (-PAGE_SIZE)
789 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
790 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
791#undef __FIXADDR_TOP
792 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
793 WARN_ON(1);
794 printk(KERN_WARNING "pmd %p != %p\n",
795 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
796 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
797 fix_to_virt(FIX_BTMAP_BEGIN));
798 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
799 fix_to_virt(FIX_BTMAP_END));
800
801 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
802 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
803 FIX_BTMAP_BEGIN);
804 }
805}
806
807void __init __early_set_fixmap(enum fixed_addresses idx,
808 phys_addr_t phys, pgprot_t flags)
809{
810 unsigned long addr = __fix_to_virt(idx);
811 pte_t *pte;
812
813 if (idx >= __end_of_fixed_addresses) {
814 BUG();
815 return;
816 }
817 pte = early_ioremap_pte(addr);
818
819 /* Sanitize 'prot' against any unsupported bits: */
820 pgprot_val(flags) &= __default_kernel_pte_mask;
821
822 if (pgprot_val(flags))
823 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
824 else
825 pte_clear(&init_mm, addr, pte);
826 __flush_tlb_one_kernel(addr);
827}