Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_POWERPC_MMU_H_
3#define _ASM_POWERPC_MMU_H_
4#ifdef __KERNEL__
5
6#include <linux/types.h>
7
8#include <asm/asm-const.h>
9
10/*
11 * MMU features bit definitions
12 */
13
14/*
15 * MMU families
16 */
17#define MMU_FTR_HPTE_TABLE ASM_CONST(0x00000001)
18#define MMU_FTR_TYPE_8xx ASM_CONST(0x00000002)
19#define MMU_FTR_TYPE_40x ASM_CONST(0x00000004)
20#define MMU_FTR_TYPE_44x ASM_CONST(0x00000008)
21#define MMU_FTR_TYPE_FSL_E ASM_CONST(0x00000010)
22#define MMU_FTR_TYPE_47x ASM_CONST(0x00000020)
23
24/* Radix page table supported and enabled */
25#define MMU_FTR_TYPE_RADIX ASM_CONST(0x00000040)
26
27/*
28 * Individual features below.
29 */
30
31/*
32 * Supports KUAP feature
33 * key 0 controlling userspace addresses on radix
34 * Key 3 on hash
35 */
36#define MMU_FTR_KUAP ASM_CONST(0x00000200)
37
38/*
39 * Supports KUEP feature
40 * key 0 controlling userspace addresses on radix
41 * Key 3 on hash
42 */
43#define MMU_FTR_BOOK3S_KUEP ASM_CONST(0x00000400)
44
45/*
46 * Support for memory protection keys.
47 */
48#define MMU_FTR_PKEY ASM_CONST(0x00000800)
49
50/* Guest Translation Shootdown Enable */
51#define MMU_FTR_GTSE ASM_CONST(0x00001000)
52
53/*
54 * Support for 68 bit VA space. We added that from ISA 2.05
55 */
56#define MMU_FTR_68_BIT_VA ASM_CONST(0x00002000)
57/*
58 * Kernel read only support.
59 * We added the ppp value 0b110 in ISA 2.04.
60 */
61#define MMU_FTR_KERNEL_RO ASM_CONST(0x00004000)
62
63/*
64 * We need to clear top 16bits of va (from the remaining 64 bits )in
65 * tlbie* instructions
66 */
67#define MMU_FTR_TLBIE_CROP_VA ASM_CONST(0x00008000)
68
69/* Enable use of high BAT registers */
70#define MMU_FTR_USE_HIGH_BATS ASM_CONST(0x00010000)
71
72/* Enable >32-bit physical addresses on 32-bit processor, only used
73 * by CONFIG_PPC_BOOK3S_32 currently as BookE supports that from day 1
74 */
75#define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000)
76
77/* Enable use of broadcast TLB invalidations. We don't always set it
78 * on processors that support it due to other constraints with the
79 * use of such invalidations
80 */
81#define MMU_FTR_USE_TLBIVAX_BCAST ASM_CONST(0x00040000)
82
83/* Enable use of tlbilx invalidate instructions.
84 */
85#define MMU_FTR_USE_TLBILX ASM_CONST(0x00080000)
86
87/* This indicates that the processor cannot handle multiple outstanding
88 * broadcast tlbivax or tlbsync. This makes the code use a spinlock
89 * around such invalidate forms.
90 */
91#define MMU_FTR_LOCK_BCAST_INVAL ASM_CONST(0x00100000)
92
93/* This indicates that the processor doesn't handle way selection
94 * properly and needs SW to track and update the LRU state. This
95 * is specific to an errata on e300c2/c3/c4 class parts
96 */
97#define MMU_FTR_NEED_DTLB_SW_LRU ASM_CONST(0x00200000)
98
99/* Doesn't support the B bit (1T segment) in SLBIE
100 */
101#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x02000000)
102
103/* Support 16M large pages
104 */
105#define MMU_FTR_16M_PAGE ASM_CONST(0x04000000)
106
107/* Supports TLBIEL variant
108 */
109#define MMU_FTR_TLBIEL ASM_CONST(0x08000000)
110
111/* Supports tlbies w/o locking
112 */
113#define MMU_FTR_LOCKLESS_TLBIE ASM_CONST(0x10000000)
114
115/* Large pages can be marked CI
116 */
117#define MMU_FTR_CI_LARGE_PAGE ASM_CONST(0x20000000)
118
119/* 1T segments available
120 */
121#define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000)
122
123// NX paste RMA reject in DSI
124#define MMU_FTR_NX_DSI ASM_CONST(0x80000000)
125
126/* MMU feature bit sets for various CPUs */
127#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 (MMU_FTR_HPTE_TABLE | MMU_FTR_TLBIEL | MMU_FTR_16M_PAGE)
128#define MMU_FTRS_POWER MMU_FTRS_DEFAULT_HPTE_ARCH_V2
129#define MMU_FTRS_PPC970 MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA
130#define MMU_FTRS_POWER5 MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE
131#define MMU_FTRS_POWER6 MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA
132#define MMU_FTRS_POWER7 MMU_FTRS_POWER6
133#define MMU_FTRS_POWER8 MMU_FTRS_POWER6
134#define MMU_FTRS_POWER9 MMU_FTRS_POWER6
135#define MMU_FTRS_POWER10 MMU_FTRS_POWER6
136#define MMU_FTRS_POWER11 MMU_FTRS_POWER6
137#define MMU_FTRS_CELL MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
138 MMU_FTR_CI_LARGE_PAGE
139#define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
140 MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
141#ifndef __ASSEMBLY__
142#include <linux/bug.h>
143#include <asm/cputable.h>
144#include <asm/page.h>
145
146typedef pte_t *pgtable_t;
147
148enum {
149 MMU_FTRS_POSSIBLE =
150#if defined(CONFIG_PPC_BOOK3S_604)
151 MMU_FTR_HPTE_TABLE |
152#endif
153#ifdef CONFIG_PPC_8xx
154 MMU_FTR_TYPE_8xx |
155#endif
156#ifdef CONFIG_40x
157 MMU_FTR_TYPE_40x |
158#endif
159#ifdef CONFIG_PPC_47x
160 MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL |
161#elif defined(CONFIG_44x)
162 MMU_FTR_TYPE_44x |
163#endif
164#ifdef CONFIG_PPC_E500
165 MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX |
166#endif
167#ifdef CONFIG_PPC_BOOK3S_32
168 MMU_FTR_USE_HIGH_BATS |
169#endif
170#ifdef CONFIG_PPC_83xx
171 MMU_FTR_NEED_DTLB_SW_LRU |
172#endif
173#ifdef CONFIG_PPC_BOOK3S_64
174 MMU_FTR_KERNEL_RO |
175#ifdef CONFIG_PPC_64S_HASH_MMU
176 MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
177 MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
178 MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
179 MMU_FTR_68_BIT_VA | MMU_FTR_HPTE_TABLE |
180#endif
181#ifdef CONFIG_PPC_RADIX_MMU
182 MMU_FTR_TYPE_RADIX |
183 MMU_FTR_GTSE | MMU_FTR_NX_DSI |
184#endif /* CONFIG_PPC_RADIX_MMU */
185#endif
186#ifdef CONFIG_PPC_KUAP
187 MMU_FTR_KUAP |
188#endif /* CONFIG_PPC_KUAP */
189#ifdef CONFIG_PPC_MEM_KEYS
190 MMU_FTR_PKEY |
191#endif
192#ifdef CONFIG_PPC_KUEP
193 MMU_FTR_BOOK3S_KUEP |
194#endif /* CONFIG_PPC_KUAP */
195
196 0,
197};
198
199#if defined(CONFIG_PPC_BOOK3S_604) && !defined(CONFIG_PPC_BOOK3S_603)
200#define MMU_FTRS_ALWAYS MMU_FTR_HPTE_TABLE
201#endif
202#ifdef CONFIG_PPC_8xx
203#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_8xx
204#endif
205#ifdef CONFIG_40x
206#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_40x
207#endif
208#ifdef CONFIG_PPC_47x
209#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_47x
210#elif defined(CONFIG_44x)
211#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_44x
212#endif
213#ifdef CONFIG_PPC_E500
214#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_FSL_E
215#endif
216
217/* BOOK3S_64 options */
218#if defined(CONFIG_PPC_RADIX_MMU) && !defined(CONFIG_PPC_64S_HASH_MMU)
219#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_RADIX
220#elif !defined(CONFIG_PPC_RADIX_MMU) && defined(CONFIG_PPC_64S_HASH_MMU)
221#define MMU_FTRS_ALWAYS MMU_FTR_HPTE_TABLE
222#endif
223
224#ifndef MMU_FTRS_ALWAYS
225#define MMU_FTRS_ALWAYS 0
226#endif
227
228static __always_inline bool early_mmu_has_feature(unsigned long feature)
229{
230 if (MMU_FTRS_ALWAYS & feature)
231 return true;
232
233 return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
234}
235
236#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
237#include <linux/jump_label.h>
238
239#define NUM_MMU_FTR_KEYS 32
240
241extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
242
243extern void mmu_feature_keys_init(void);
244
245static __always_inline bool mmu_has_feature(unsigned long feature)
246{
247 int i;
248
249#ifndef __clang__ /* clang can't cope with this */
250 BUILD_BUG_ON(!__builtin_constant_p(feature));
251#endif
252
253#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
254 if (!static_key_initialized) {
255 printk("Warning! mmu_has_feature() used prior to jump label init!\n");
256 dump_stack();
257 return early_mmu_has_feature(feature);
258 }
259#endif
260
261 if (MMU_FTRS_ALWAYS & feature)
262 return true;
263
264 if (!(MMU_FTRS_POSSIBLE & feature))
265 return false;
266
267 i = __builtin_ctzl(feature);
268 return static_branch_likely(&mmu_feature_keys[i]);
269}
270
271static inline void mmu_clear_feature(unsigned long feature)
272{
273 int i;
274
275 i = __builtin_ctzl(feature);
276 cur_cpu_spec->mmu_features &= ~feature;
277 static_branch_disable(&mmu_feature_keys[i]);
278}
279#else
280
281static inline void mmu_feature_keys_init(void)
282{
283
284}
285
286static __always_inline bool mmu_has_feature(unsigned long feature)
287{
288 return early_mmu_has_feature(feature);
289}
290
291static inline void mmu_clear_feature(unsigned long feature)
292{
293 cur_cpu_spec->mmu_features &= ~feature;
294}
295#endif /* CONFIG_JUMP_LABEL */
296
297extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
298
299#ifdef CONFIG_PPC64
300/* This is our real memory area size on ppc64 server, on embedded, we
301 * make it match the size our of bolted TLB area
302 */
303extern u64 ppc64_rma_size;
304
305/* Cleanup function used by kexec */
306extern void mmu_cleanup_all(void);
307extern void radix__mmu_cleanup_all(void);
308
309/* Functions for creating and updating partition table on POWER9 */
310extern void mmu_partition_table_init(void);
311extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
312 unsigned long dw1, bool flush);
313#endif /* CONFIG_PPC64 */
314
315struct mm_struct;
316#ifdef CONFIG_DEBUG_VM
317extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
318#else /* CONFIG_DEBUG_VM */
319static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
320{
321}
322#endif /* !CONFIG_DEBUG_VM */
323
324static __always_inline bool radix_enabled(void)
325{
326 return mmu_has_feature(MMU_FTR_TYPE_RADIX);
327}
328
329static __always_inline bool early_radix_enabled(void)
330{
331 return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
332}
333
334static inline bool strict_kernel_rwx_enabled(void)
335{
336 return IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled;
337}
338
339static inline bool strict_module_rwx_enabled(void)
340{
341 return IS_ENABLED(CONFIG_STRICT_MODULE_RWX) && strict_kernel_rwx_enabled();
342}
343#endif /* !__ASSEMBLY__ */
344
345/* The kernel use the constants below to index in the page sizes array.
346 * The use of fixed constants for this purpose is better for performances
347 * of the low level hash refill handlers.
348 *
349 * A non supported page size has a "shift" field set to 0
350 *
351 * Any new page size being implemented can get a new entry in here. Whether
352 * the kernel will use it or not is a different matter though. The actual page
353 * size used by hugetlbfs is not defined here and may be made variable
354 *
355 * Note: This array ended up being a false good idea as it's growing to the
356 * point where I wonder if we should replace it with something different,
357 * to think about, feedback welcome. --BenH.
358 */
359
360/* These are #defines as they have to be used in assembly */
361#define MMU_PAGE_4K 0
362#define MMU_PAGE_16K 1
363#define MMU_PAGE_64K 2
364#define MMU_PAGE_64K_AP 3 /* "Admixed pages" (hash64 only) */
365#define MMU_PAGE_256K 4
366#define MMU_PAGE_512K 5
367#define MMU_PAGE_1M 6
368#define MMU_PAGE_2M 7
369#define MMU_PAGE_4M 8
370#define MMU_PAGE_8M 9
371#define MMU_PAGE_16M 10
372#define MMU_PAGE_64M 11
373#define MMU_PAGE_256M 12
374#define MMU_PAGE_1G 13
375#define MMU_PAGE_16G 14
376#define MMU_PAGE_64G 15
377
378/*
379 * N.B. we need to change the type of hpte_page_sizes if this gets to be > 16
380 * Also we need to change he type of mm_context.low/high_slices_psize.
381 */
382#define MMU_PAGE_COUNT 16
383
384#ifdef CONFIG_PPC_BOOK3S_64
385#include <asm/book3s/64/mmu.h>
386#else /* CONFIG_PPC_BOOK3S_64 */
387
388#ifndef __ASSEMBLY__
389/* MMU initialization */
390extern void early_init_mmu(void);
391extern void early_init_mmu_secondary(void);
392extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
393 phys_addr_t first_memblock_size);
394static inline void mmu_early_init_devtree(void) { }
395
396static inline void pkey_early_init_devtree(void) {}
397
398extern void *abatron_pteptrs[2];
399#endif /* __ASSEMBLY__ */
400#endif
401
402#if defined(CONFIG_PPC_BOOK3S_32)
403/* 32-bit classic hash table MMU */
404#include <asm/book3s/32/mmu-hash.h>
405#elif defined(CONFIG_PPC_MMU_NOHASH)
406#include <asm/nohash/mmu.h>
407#endif
408
409#if defined(CONFIG_FA_DUMP) || defined(CONFIG_PRESERVE_FA_DUMP)
410#define __HAVE_ARCH_RESERVED_KERNEL_PAGES
411#endif
412
413#endif /* __KERNEL__ */
414#endif /* _ASM_POWERPC_MMU_H_ */
1#ifndef _ASM_POWERPC_MMU_H_
2#define _ASM_POWERPC_MMU_H_
3#ifdef __KERNEL__
4
5#include <linux/types.h>
6
7#include <asm/asm-compat.h>
8#include <asm/feature-fixups.h>
9
10/*
11 * MMU features bit definitions
12 */
13
14/*
15 * MMU families
16 */
17#define MMU_FTR_HPTE_TABLE ASM_CONST(0x00000001)
18#define MMU_FTR_TYPE_8xx ASM_CONST(0x00000002)
19#define MMU_FTR_TYPE_40x ASM_CONST(0x00000004)
20#define MMU_FTR_TYPE_44x ASM_CONST(0x00000008)
21#define MMU_FTR_TYPE_FSL_E ASM_CONST(0x00000010)
22#define MMU_FTR_TYPE_47x ASM_CONST(0x00000020)
23
24/* Radix page table supported and enabled */
25#define MMU_FTR_TYPE_RADIX ASM_CONST(0x00000040)
26
27/*
28 * Individual features below.
29 */
30
31/*
32 * Kernel read only support.
33 * We added the ppp value 0b110 in ISA 2.04.
34 */
35#define MMU_FTR_KERNEL_RO ASM_CONST(0x00004000)
36
37/*
38 * We need to clear top 16bits of va (from the remaining 64 bits )in
39 * tlbie* instructions
40 */
41#define MMU_FTR_TLBIE_CROP_VA ASM_CONST(0x00008000)
42
43/* Enable use of high BAT registers */
44#define MMU_FTR_USE_HIGH_BATS ASM_CONST(0x00010000)
45
46/* Enable >32-bit physical addresses on 32-bit processor, only used
47 * by CONFIG_6xx currently as BookE supports that from day 1
48 */
49#define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000)
50
51/* Enable use of broadcast TLB invalidations. We don't always set it
52 * on processors that support it due to other constraints with the
53 * use of such invalidations
54 */
55#define MMU_FTR_USE_TLBIVAX_BCAST ASM_CONST(0x00040000)
56
57/* Enable use of tlbilx invalidate instructions.
58 */
59#define MMU_FTR_USE_TLBILX ASM_CONST(0x00080000)
60
61/* This indicates that the processor cannot handle multiple outstanding
62 * broadcast tlbivax or tlbsync. This makes the code use a spinlock
63 * around such invalidate forms.
64 */
65#define MMU_FTR_LOCK_BCAST_INVAL ASM_CONST(0x00100000)
66
67/* This indicates that the processor doesn't handle way selection
68 * properly and needs SW to track and update the LRU state. This
69 * is specific to an errata on e300c2/c3/c4 class parts
70 */
71#define MMU_FTR_NEED_DTLB_SW_LRU ASM_CONST(0x00200000)
72
73/* Enable use of TLB reservation. Processor should support tlbsrx.
74 * instruction and MAS0[WQ].
75 */
76#define MMU_FTR_USE_TLBRSRV ASM_CONST(0x00800000)
77
78/* Use paired MAS registers (MAS7||MAS3, etc.)
79 */
80#define MMU_FTR_USE_PAIRED_MAS ASM_CONST(0x01000000)
81
82/* Doesn't support the B bit (1T segment) in SLBIE
83 */
84#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x02000000)
85
86/* Support 16M large pages
87 */
88#define MMU_FTR_16M_PAGE ASM_CONST(0x04000000)
89
90/* Supports TLBIEL variant
91 */
92#define MMU_FTR_TLBIEL ASM_CONST(0x08000000)
93
94/* Supports tlbies w/o locking
95 */
96#define MMU_FTR_LOCKLESS_TLBIE ASM_CONST(0x10000000)
97
98/* Large pages can be marked CI
99 */
100#define MMU_FTR_CI_LARGE_PAGE ASM_CONST(0x20000000)
101
102/* 1T segments available
103 */
104#define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000)
105
106/* MMU feature bit sets for various CPUs */
107#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 \
108 MMU_FTR_HPTE_TABLE | MMU_FTR_PPCAS_ARCH_V2
109#define MMU_FTRS_POWER4 MMU_FTRS_DEFAULT_HPTE_ARCH_V2
110#define MMU_FTRS_PPC970 MMU_FTRS_POWER4 | MMU_FTR_TLBIE_CROP_VA
111#define MMU_FTRS_POWER5 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE
112#define MMU_FTRS_POWER6 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_KERNEL_RO
113#define MMU_FTRS_POWER7 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_KERNEL_RO
114#define MMU_FTRS_POWER8 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_KERNEL_RO
115#define MMU_FTRS_POWER9 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_KERNEL_RO
116#define MMU_FTRS_CELL MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
117 MMU_FTR_CI_LARGE_PAGE
118#define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
119 MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
120#ifndef __ASSEMBLY__
121#include <linux/bug.h>
122#include <asm/cputable.h>
123
124#ifdef CONFIG_PPC_FSL_BOOK3E
125#include <asm/percpu.h>
126DECLARE_PER_CPU(int, next_tlbcam_idx);
127#endif
128
129enum {
130 MMU_FTRS_POSSIBLE = MMU_FTR_HPTE_TABLE | MMU_FTR_TYPE_8xx |
131 MMU_FTR_TYPE_40x | MMU_FTR_TYPE_44x | MMU_FTR_TYPE_FSL_E |
132 MMU_FTR_TYPE_47x | MMU_FTR_USE_HIGH_BATS | MMU_FTR_BIG_PHYS |
133 MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_USE_TLBILX |
134 MMU_FTR_LOCK_BCAST_INVAL | MMU_FTR_NEED_DTLB_SW_LRU |
135 MMU_FTR_USE_TLBRSRV | MMU_FTR_USE_PAIRED_MAS |
136 MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
137 MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
138 MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
139 MMU_FTR_KERNEL_RO |
140#ifdef CONFIG_PPC_RADIX_MMU
141 MMU_FTR_TYPE_RADIX |
142#endif
143 0,
144};
145
146static inline bool early_mmu_has_feature(unsigned long feature)
147{
148 return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
149}
150
151#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
152#include <linux/jump_label.h>
153
154#define NUM_MMU_FTR_KEYS 32
155
156extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
157
158extern void mmu_feature_keys_init(void);
159
160static __always_inline bool mmu_has_feature(unsigned long feature)
161{
162 int i;
163
164#ifndef __clang__ /* clang can't cope with this */
165 BUILD_BUG_ON(!__builtin_constant_p(feature));
166#endif
167
168#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
169 if (!static_key_initialized) {
170 printk("Warning! mmu_has_feature() used prior to jump label init!\n");
171 dump_stack();
172 return early_mmu_has_feature(feature);
173 }
174#endif
175
176 if (!(MMU_FTRS_POSSIBLE & feature))
177 return false;
178
179 i = __builtin_ctzl(feature);
180 return static_branch_likely(&mmu_feature_keys[i]);
181}
182
183static inline void mmu_clear_feature(unsigned long feature)
184{
185 int i;
186
187 i = __builtin_ctzl(feature);
188 cur_cpu_spec->mmu_features &= ~feature;
189 static_branch_disable(&mmu_feature_keys[i]);
190}
191#else
192
193static inline void mmu_feature_keys_init(void)
194{
195
196}
197
198static inline bool mmu_has_feature(unsigned long feature)
199{
200 return early_mmu_has_feature(feature);
201}
202
203static inline void mmu_clear_feature(unsigned long feature)
204{
205 cur_cpu_spec->mmu_features &= ~feature;
206}
207#endif /* CONFIG_JUMP_LABEL */
208
209extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
210
211#ifdef CONFIG_PPC64
212/* This is our real memory area size on ppc64 server, on embedded, we
213 * make it match the size our of bolted TLB area
214 */
215extern u64 ppc64_rma_size;
216
217/* Cleanup function used by kexec */
218extern void mmu_cleanup_all(void);
219extern void radix__mmu_cleanup_all(void);
220
221/* Functions for creating and updating partition table on POWER9 */
222extern void mmu_partition_table_init(void);
223extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
224 unsigned long dw1);
225#endif /* CONFIG_PPC64 */
226
227struct mm_struct;
228#ifdef CONFIG_DEBUG_VM
229extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
230#else /* CONFIG_DEBUG_VM */
231static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
232{
233}
234#endif /* !CONFIG_DEBUG_VM */
235
236#ifdef CONFIG_PPC_RADIX_MMU
237static inline bool radix_enabled(void)
238{
239 return mmu_has_feature(MMU_FTR_TYPE_RADIX);
240}
241
242static inline bool early_radix_enabled(void)
243{
244 return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
245}
246#else
247static inline bool radix_enabled(void)
248{
249 return false;
250}
251
252static inline bool early_radix_enabled(void)
253{
254 return false;
255}
256#endif
257
258#endif /* !__ASSEMBLY__ */
259
260/* The kernel use the constants below to index in the page sizes array.
261 * The use of fixed constants for this purpose is better for performances
262 * of the low level hash refill handlers.
263 *
264 * A non supported page size has a "shift" field set to 0
265 *
266 * Any new page size being implemented can get a new entry in here. Whether
267 * the kernel will use it or not is a different matter though. The actual page
268 * size used by hugetlbfs is not defined here and may be made variable
269 *
270 * Note: This array ended up being a false good idea as it's growing to the
271 * point where I wonder if we should replace it with something different,
272 * to think about, feedback welcome. --BenH.
273 */
274
275/* These are #defines as they have to be used in assembly */
276#define MMU_PAGE_4K 0
277#define MMU_PAGE_16K 1
278#define MMU_PAGE_64K 2
279#define MMU_PAGE_64K_AP 3 /* "Admixed pages" (hash64 only) */
280#define MMU_PAGE_256K 4
281#define MMU_PAGE_512K 5
282#define MMU_PAGE_1M 6
283#define MMU_PAGE_2M 7
284#define MMU_PAGE_4M 8
285#define MMU_PAGE_8M 9
286#define MMU_PAGE_16M 10
287#define MMU_PAGE_64M 11
288#define MMU_PAGE_256M 12
289#define MMU_PAGE_1G 13
290#define MMU_PAGE_16G 14
291#define MMU_PAGE_64G 15
292
293/* N.B. we need to change the type of hpte_page_sizes if this gets to be > 16 */
294#define MMU_PAGE_COUNT 16
295
296#ifdef CONFIG_PPC_BOOK3S_64
297#include <asm/book3s/64/mmu.h>
298#else /* CONFIG_PPC_BOOK3S_64 */
299
300#ifndef __ASSEMBLY__
301/* MMU initialization */
302extern void early_init_mmu(void);
303extern void early_init_mmu_secondary(void);
304extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
305 phys_addr_t first_memblock_size);
306static inline void mmu_early_init_devtree(void) { }
307#endif /* __ASSEMBLY__ */
308#endif
309
310#if defined(CONFIG_PPC_STD_MMU_32)
311/* 32-bit classic hash table MMU */
312#include <asm/book3s/32/mmu-hash.h>
313#elif defined(CONFIG_40x)
314/* 40x-style software loaded TLB */
315# include <asm/mmu-40x.h>
316#elif defined(CONFIG_44x)
317/* 44x-style software loaded TLB */
318# include <asm/mmu-44x.h>
319#elif defined(CONFIG_PPC_BOOK3E_MMU)
320/* Freescale Book-E software loaded TLB or Book-3e (ISA 2.06+) MMU */
321# include <asm/mmu-book3e.h>
322#elif defined (CONFIG_PPC_8xx)
323/* Motorola/Freescale 8xx software loaded TLB */
324# include <asm/mmu-8xx.h>
325#endif
326
327#endif /* __KERNEL__ */
328#endif /* _ASM_POWERPC_MMU_H_ */