Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * This file contains pgtable related functions for 64-bit machines.
4 *
5 * Derived from arch/ppc64/mm/init.c
6 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
7 *
8 * Modifications by Paul Mackerras (PowerMac) (paulus@samba.org)
9 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
10 * Copyright (C) 1996 Paul Mackerras
11 *
12 * Derived from "arch/i386/mm/init.c"
13 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
14 *
15 * Dave Engebretsen <engebret@us.ibm.com>
16 * Rework for PPC64 port.
17 */
18
19#include <linux/signal.h>
20#include <linux/sched.h>
21#include <linux/kernel.h>
22#include <linux/errno.h>
23#include <linux/string.h>
24#include <linux/export.h>
25#include <linux/types.h>
26#include <linux/mman.h>
27#include <linux/mm.h>
28#include <linux/swap.h>
29#include <linux/stddef.h>
30#include <linux/vmalloc.h>
31#include <linux/slab.h>
32#include <linux/hugetlb.h>
33
34#include <asm/page.h>
35#include <asm/mmu_context.h>
36#include <asm/mmu.h>
37#include <asm/smp.h>
38#include <asm/machdep.h>
39#include <asm/tlb.h>
40#include <asm/processor.h>
41#include <asm/cputable.h>
42#include <asm/sections.h>
43#include <asm/firmware.h>
44#include <asm/dma.h>
45
46#include <mm/mmu_decl.h>
47
48
49#ifdef CONFIG_PPC_BOOK3S_64
50/*
51 * partition table and process table for ISA 3.0
52 */
53struct prtb_entry *process_tb;
54struct patb_entry *partition_tb;
55/*
56 * page table size
57 */
58unsigned long __pte_index_size;
59EXPORT_SYMBOL(__pte_index_size);
60unsigned long __pmd_index_size;
61EXPORT_SYMBOL(__pmd_index_size);
62unsigned long __pud_index_size;
63EXPORT_SYMBOL(__pud_index_size);
64unsigned long __pgd_index_size;
65EXPORT_SYMBOL(__pgd_index_size);
66unsigned long __pud_cache_index;
67EXPORT_SYMBOL(__pud_cache_index);
68unsigned long __pte_table_size;
69EXPORT_SYMBOL(__pte_table_size);
70unsigned long __pmd_table_size;
71EXPORT_SYMBOL(__pmd_table_size);
72unsigned long __pud_table_size;
73EXPORT_SYMBOL(__pud_table_size);
74unsigned long __pgd_table_size;
75EXPORT_SYMBOL(__pgd_table_size);
76unsigned long __pmd_val_bits;
77EXPORT_SYMBOL(__pmd_val_bits);
78unsigned long __pud_val_bits;
79EXPORT_SYMBOL(__pud_val_bits);
80unsigned long __pgd_val_bits;
81EXPORT_SYMBOL(__pgd_val_bits);
82unsigned long __kernel_virt_start;
83EXPORT_SYMBOL(__kernel_virt_start);
84unsigned long __vmalloc_start;
85EXPORT_SYMBOL(__vmalloc_start);
86unsigned long __vmalloc_end;
87EXPORT_SYMBOL(__vmalloc_end);
88unsigned long __kernel_io_start;
89EXPORT_SYMBOL(__kernel_io_start);
90unsigned long __kernel_io_end;
91struct page *vmemmap;
92EXPORT_SYMBOL(vmemmap);
93unsigned long __pte_frag_nr;
94EXPORT_SYMBOL(__pte_frag_nr);
95unsigned long __pte_frag_size_shift;
96EXPORT_SYMBOL(__pte_frag_size_shift);
97#endif
98
99#ifndef __PAGETABLE_PUD_FOLDED
100/* 4 level page table */
101struct page *p4d_page(p4d_t p4d)
102{
103 if (p4d_is_leaf(p4d)) {
104 if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
105 VM_WARN_ON(!p4d_huge(p4d));
106 return pte_page(p4d_pte(p4d));
107 }
108 return virt_to_page(p4d_pgtable(p4d));
109}
110#endif
111
112struct page *pud_page(pud_t pud)
113{
114 if (pud_is_leaf(pud)) {
115 if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
116 VM_WARN_ON(!pud_huge(pud));
117 return pte_page(pud_pte(pud));
118 }
119 return virt_to_page(pud_pgtable(pud));
120}
121
122/*
123 * For hugepage we have pfn in the pmd, we use PTE_RPN_SHIFT bits for flags
124 * For PTE page, we have a PTE_FRAG_SIZE (4K) aligned virtual address.
125 */
126struct page *pmd_page(pmd_t pmd)
127{
128 if (pmd_is_leaf(pmd)) {
129 /*
130 * vmalloc_to_page may be called on any vmap address (not only
131 * vmalloc), and it uses pmd_page() etc., when huge vmap is
132 * enabled so these checks can't be used.
133 */
134 if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
135 VM_WARN_ON(!(pmd_large(pmd) || pmd_huge(pmd)));
136 return pte_page(pmd_pte(pmd));
137 }
138 return virt_to_page(pmd_page_vaddr(pmd));
139}
140
141#ifdef CONFIG_STRICT_KERNEL_RWX
142void mark_rodata_ro(void)
143{
144 if (!mmu_has_feature(MMU_FTR_KERNEL_RO)) {
145 pr_warn("Warning: Unable to mark rodata read only on this CPU.\n");
146 return;
147 }
148
149 if (radix_enabled())
150 radix__mark_rodata_ro();
151 else
152 hash__mark_rodata_ro();
153
154 // mark_initmem_nx() should have already run by now
155 ptdump_check_wx();
156}
157
158void mark_initmem_nx(void)
159{
160 if (radix_enabled())
161 radix__mark_initmem_nx();
162 else
163 hash__mark_initmem_nx();
164}
165#endif
1/*
2 * This file contains ioremap and related functions for 64-bit machines.
3 *
4 * Derived from arch/ppc64/mm/init.c
5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
6 *
7 * Modifications by Paul Mackerras (PowerMac) (paulus@samba.org)
8 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
9 * Copyright (C) 1996 Paul Mackerras
10 *
11 * Derived from "arch/i386/mm/init.c"
12 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
13 *
14 * Dave Engebretsen <engebret@us.ibm.com>
15 * Rework for PPC64 port.
16 *
17 * This program is free software; you can redistribute it and/or
18 * modify it under the terms of the GNU General Public License
19 * as published by the Free Software Foundation; either version
20 * 2 of the License, or (at your option) any later version.
21 *
22 */
23
24#include <linux/signal.h>
25#include <linux/sched.h>
26#include <linux/kernel.h>
27#include <linux/errno.h>
28#include <linux/string.h>
29#include <linux/export.h>
30#include <linux/types.h>
31#include <linux/mman.h>
32#include <linux/mm.h>
33#include <linux/swap.h>
34#include <linux/stddef.h>
35#include <linux/vmalloc.h>
36#include <linux/memblock.h>
37#include <linux/slab.h>
38#include <linux/hugetlb.h>
39
40#include <asm/pgalloc.h>
41#include <asm/page.h>
42#include <asm/prom.h>
43#include <asm/io.h>
44#include <asm/mmu_context.h>
45#include <asm/pgtable.h>
46#include <asm/mmu.h>
47#include <asm/smp.h>
48#include <asm/machdep.h>
49#include <asm/tlb.h>
50#include <asm/processor.h>
51#include <asm/cputable.h>
52#include <asm/sections.h>
53#include <asm/firmware.h>
54#include <asm/dma.h>
55
56#include "mmu_decl.h"
57
58#ifdef CONFIG_PPC_STD_MMU_64
59#if TASK_SIZE_USER64 > (1UL << (ESID_BITS + SID_SHIFT))
60#error TASK_SIZE_USER64 exceeds user VSID range
61#endif
62#endif
63
64#ifdef CONFIG_PPC_BOOK3S_64
65/*
66 * partition table and process table for ISA 3.0
67 */
68struct prtb_entry *process_tb;
69struct patb_entry *partition_tb;
70/*
71 * page table size
72 */
73unsigned long __pte_index_size;
74EXPORT_SYMBOL(__pte_index_size);
75unsigned long __pmd_index_size;
76EXPORT_SYMBOL(__pmd_index_size);
77unsigned long __pud_index_size;
78EXPORT_SYMBOL(__pud_index_size);
79unsigned long __pgd_index_size;
80EXPORT_SYMBOL(__pgd_index_size);
81unsigned long __pmd_cache_index;
82EXPORT_SYMBOL(__pmd_cache_index);
83unsigned long __pte_table_size;
84EXPORT_SYMBOL(__pte_table_size);
85unsigned long __pmd_table_size;
86EXPORT_SYMBOL(__pmd_table_size);
87unsigned long __pud_table_size;
88EXPORT_SYMBOL(__pud_table_size);
89unsigned long __pgd_table_size;
90EXPORT_SYMBOL(__pgd_table_size);
91unsigned long __pmd_val_bits;
92EXPORT_SYMBOL(__pmd_val_bits);
93unsigned long __pud_val_bits;
94EXPORT_SYMBOL(__pud_val_bits);
95unsigned long __pgd_val_bits;
96EXPORT_SYMBOL(__pgd_val_bits);
97unsigned long __kernel_virt_start;
98EXPORT_SYMBOL(__kernel_virt_start);
99unsigned long __kernel_virt_size;
100EXPORT_SYMBOL(__kernel_virt_size);
101unsigned long __vmalloc_start;
102EXPORT_SYMBOL(__vmalloc_start);
103unsigned long __vmalloc_end;
104EXPORT_SYMBOL(__vmalloc_end);
105struct page *vmemmap;
106EXPORT_SYMBOL(vmemmap);
107unsigned long __pte_frag_nr;
108EXPORT_SYMBOL(__pte_frag_nr);
109unsigned long __pte_frag_size_shift;
110EXPORT_SYMBOL(__pte_frag_size_shift);
111unsigned long ioremap_bot;
112#else /* !CONFIG_PPC_BOOK3S_64 */
113unsigned long ioremap_bot = IOREMAP_BASE;
114#endif
115
116/**
117 * __ioremap_at - Low level function to establish the page tables
118 * for an IO mapping
119 */
120void __iomem * __ioremap_at(phys_addr_t pa, void *ea, unsigned long size,
121 unsigned long flags)
122{
123 unsigned long i;
124
125 /* Make sure we have the base flags */
126 if ((flags & _PAGE_PRESENT) == 0)
127 flags |= pgprot_val(PAGE_KERNEL);
128
129 /* We don't support the 4K PFN hack with ioremap */
130 if (flags & H_PAGE_4K_PFN)
131 return NULL;
132
133 WARN_ON(pa & ~PAGE_MASK);
134 WARN_ON(((unsigned long)ea) & ~PAGE_MASK);
135 WARN_ON(size & ~PAGE_MASK);
136
137 for (i = 0; i < size; i += PAGE_SIZE)
138 if (map_kernel_page((unsigned long)ea+i, pa+i, flags))
139 return NULL;
140
141 return (void __iomem *)ea;
142}
143
144/**
145 * __iounmap_from - Low level function to tear down the page tables
146 * for an IO mapping. This is used for mappings that
147 * are manipulated manually, like partial unmapping of
148 * PCI IOs or ISA space.
149 */
150void __iounmap_at(void *ea, unsigned long size)
151{
152 WARN_ON(((unsigned long)ea) & ~PAGE_MASK);
153 WARN_ON(size & ~PAGE_MASK);
154
155 unmap_kernel_range((unsigned long)ea, size);
156}
157
158void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size,
159 unsigned long flags, void *caller)
160{
161 phys_addr_t paligned;
162 void __iomem *ret;
163
164 /*
165 * Choose an address to map it to.
166 * Once the imalloc system is running, we use it.
167 * Before that, we map using addresses going
168 * up from ioremap_bot. imalloc will use
169 * the addresses from ioremap_bot through
170 * IMALLOC_END
171 *
172 */
173 paligned = addr & PAGE_MASK;
174 size = PAGE_ALIGN(addr + size) - paligned;
175
176 if ((size == 0) || (paligned == 0))
177 return NULL;
178
179 if (slab_is_available()) {
180 struct vm_struct *area;
181
182 area = __get_vm_area_caller(size, VM_IOREMAP,
183 ioremap_bot, IOREMAP_END,
184 caller);
185 if (area == NULL)
186 return NULL;
187
188 area->phys_addr = paligned;
189 ret = __ioremap_at(paligned, area->addr, size, flags);
190 if (!ret)
191 vunmap(area->addr);
192 } else {
193 ret = __ioremap_at(paligned, (void *)ioremap_bot, size, flags);
194 if (ret)
195 ioremap_bot += size;
196 }
197
198 if (ret)
199 ret += addr & ~PAGE_MASK;
200 return ret;
201}
202
203void __iomem * __ioremap(phys_addr_t addr, unsigned long size,
204 unsigned long flags)
205{
206 return __ioremap_caller(addr, size, flags, __builtin_return_address(0));
207}
208
209void __iomem * ioremap(phys_addr_t addr, unsigned long size)
210{
211 unsigned long flags = pgprot_val(pgprot_noncached(__pgprot(0)));
212 void *caller = __builtin_return_address(0);
213
214 if (ppc_md.ioremap)
215 return ppc_md.ioremap(addr, size, flags, caller);
216 return __ioremap_caller(addr, size, flags, caller);
217}
218
219void __iomem * ioremap_wc(phys_addr_t addr, unsigned long size)
220{
221 unsigned long flags = pgprot_val(pgprot_noncached_wc(__pgprot(0)));
222 void *caller = __builtin_return_address(0);
223
224 if (ppc_md.ioremap)
225 return ppc_md.ioremap(addr, size, flags, caller);
226 return __ioremap_caller(addr, size, flags, caller);
227}
228
229void __iomem * ioremap_prot(phys_addr_t addr, unsigned long size,
230 unsigned long flags)
231{
232 void *caller = __builtin_return_address(0);
233
234 /* writeable implies dirty for kernel addresses */
235 if (flags & _PAGE_WRITE)
236 flags |= _PAGE_DIRTY;
237
238 /* we don't want to let _PAGE_EXEC leak out */
239 flags &= ~_PAGE_EXEC;
240 /*
241 * Force kernel mapping.
242 */
243#if defined(CONFIG_PPC_BOOK3S_64)
244 flags |= _PAGE_PRIVILEGED;
245#else
246 flags &= ~_PAGE_USER;
247#endif
248
249
250#ifdef _PAGE_BAP_SR
251 /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format
252 * which means that we just cleared supervisor access... oops ;-) This
253 * restores it
254 */
255 flags |= _PAGE_BAP_SR;
256#endif
257
258 if (ppc_md.ioremap)
259 return ppc_md.ioremap(addr, size, flags, caller);
260 return __ioremap_caller(addr, size, flags, caller);
261}
262
263
264/*
265 * Unmap an IO region and remove it from imalloc'd list.
266 * Access to IO memory should be serialized by driver.
267 */
268void __iounmap(volatile void __iomem *token)
269{
270 void *addr;
271
272 if (!slab_is_available())
273 return;
274
275 addr = (void *) ((unsigned long __force)
276 PCI_FIX_ADDR(token) & PAGE_MASK);
277 if ((unsigned long)addr < ioremap_bot) {
278 printk(KERN_WARNING "Attempt to iounmap early bolted mapping"
279 " at 0x%p\n", addr);
280 return;
281 }
282 vunmap(addr);
283}
284
285void iounmap(volatile void __iomem *token)
286{
287 if (ppc_md.iounmap)
288 ppc_md.iounmap(token);
289 else
290 __iounmap(token);
291}
292
293EXPORT_SYMBOL(ioremap);
294EXPORT_SYMBOL(ioremap_wc);
295EXPORT_SYMBOL(ioremap_prot);
296EXPORT_SYMBOL(__ioremap);
297EXPORT_SYMBOL(__ioremap_at);
298EXPORT_SYMBOL(iounmap);
299EXPORT_SYMBOL(__iounmap);
300EXPORT_SYMBOL(__iounmap_at);
301
302#ifndef __PAGETABLE_PUD_FOLDED
303/* 4 level page table */
304struct page *pgd_page(pgd_t pgd)
305{
306 if (pgd_huge(pgd))
307 return pte_page(pgd_pte(pgd));
308 return virt_to_page(pgd_page_vaddr(pgd));
309}
310#endif
311
312struct page *pud_page(pud_t pud)
313{
314 if (pud_huge(pud))
315 return pte_page(pud_pte(pud));
316 return virt_to_page(pud_page_vaddr(pud));
317}
318
319/*
320 * For hugepage we have pfn in the pmd, we use PTE_RPN_SHIFT bits for flags
321 * For PTE page, we have a PTE_FRAG_SIZE (4K) aligned virtual address.
322 */
323struct page *pmd_page(pmd_t pmd)
324{
325 if (pmd_trans_huge(pmd) || pmd_huge(pmd))
326 return pte_page(pmd_pte(pmd));
327 return virt_to_page(pmd_page_vaddr(pmd));
328}
329
330#ifdef CONFIG_PPC_64K_PAGES
331static pte_t *get_from_cache(struct mm_struct *mm)
332{
333 void *pte_frag, *ret;
334
335 spin_lock(&mm->page_table_lock);
336 ret = mm->context.pte_frag;
337 if (ret) {
338 pte_frag = ret + PTE_FRAG_SIZE;
339 /*
340 * If we have taken up all the fragments mark PTE page NULL
341 */
342 if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
343 pte_frag = NULL;
344 mm->context.pte_frag = pte_frag;
345 }
346 spin_unlock(&mm->page_table_lock);
347 return (pte_t *)ret;
348}
349
350static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
351{
352 void *ret = NULL;
353 struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO);
354 if (!page)
355 return NULL;
356 if (!kernel && !pgtable_page_ctor(page)) {
357 __free_page(page);
358 return NULL;
359 }
360
361 ret = page_address(page);
362 spin_lock(&mm->page_table_lock);
363 /*
364 * If we find pgtable_page set, we return
365 * the allocated page with single fragement
366 * count.
367 */
368 if (likely(!mm->context.pte_frag)) {
369 set_page_count(page, PTE_FRAG_NR);
370 mm->context.pte_frag = ret + PTE_FRAG_SIZE;
371 }
372 spin_unlock(&mm->page_table_lock);
373
374 return (pte_t *)ret;
375}
376
377pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
378{
379 pte_t *pte;
380
381 pte = get_from_cache(mm);
382 if (pte)
383 return pte;
384
385 return __alloc_for_cache(mm, kernel);
386}
387#endif /* CONFIG_PPC_64K_PAGES */
388
389void pte_fragment_free(unsigned long *table, int kernel)
390{
391 struct page *page = virt_to_page(table);
392 if (put_page_testzero(page)) {
393 if (!kernel)
394 pgtable_page_dtor(page);
395 free_hot_cold_page(page, 0);
396 }
397}
398
399#ifdef CONFIG_SMP
400void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
401{
402 unsigned long pgf = (unsigned long)table;
403
404 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
405 pgf |= shift;
406 tlb_remove_table(tlb, (void *)pgf);
407}
408
409void __tlb_remove_table(void *_table)
410{
411 void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
412 unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
413
414 if (!shift)
415 /* PTE page needs special handling */
416 pte_fragment_free(table, 0);
417 else {
418 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
419 kmem_cache_free(PGT_CACHE(shift), table);
420 }
421}
422#else
423void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
424{
425 if (!shift) {
426 /* PTE page needs special handling */
427 pte_fragment_free(table, 0);
428 } else {
429 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
430 kmem_cache_free(PGT_CACHE(shift), table);
431 }
432}
433#endif
434
435#ifdef CONFIG_PPC_BOOK3S_64
436void __init mmu_partition_table_init(void)
437{
438 unsigned long patb_size = 1UL << PATB_SIZE_SHIFT;
439
440 BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large.");
441 partition_tb = __va(memblock_alloc_base(patb_size, patb_size,
442 MEMBLOCK_ALLOC_ANYWHERE));
443
444 /* Initialize the Partition Table with no entries */
445 memset((void *)partition_tb, 0, patb_size);
446
447 /*
448 * update partition table control register,
449 * 64 K size.
450 */
451 mtspr(SPRN_PTCR, __pa(partition_tb) | (PATB_SIZE_SHIFT - 12));
452}
453
454void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
455 unsigned long dw1)
456{
457 partition_tb[lpid].patb0 = cpu_to_be64(dw0);
458 partition_tb[lpid].patb1 = cpu_to_be64(dw1);
459
460 /* Global flush of TLBs and partition table caches for this lpid */
461 asm volatile("ptesync" : : : "memory");
462 asm volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
463 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
464 asm volatile("eieio; tlbsync; ptesync" : : : "memory");
465}
466EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
467#endif /* CONFIG_PPC_BOOK3S_64 */