Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_POWERPC_MMU_H_
3#define _ASM_POWERPC_MMU_H_
4#ifdef __KERNEL__
5
6#include <linux/types.h>
7
8#include <asm/asm-const.h>
9
10/*
11 * MMU features bit definitions
12 */
13
14/*
15 * MMU families
16 */
17#define MMU_FTR_HPTE_TABLE ASM_CONST(0x00000001)
18#define MMU_FTR_TYPE_8xx ASM_CONST(0x00000002)
19#define MMU_FTR_TYPE_40x ASM_CONST(0x00000004)
20#define MMU_FTR_TYPE_44x ASM_CONST(0x00000008)
21#define MMU_FTR_TYPE_FSL_E ASM_CONST(0x00000010)
22#define MMU_FTR_TYPE_47x ASM_CONST(0x00000020)
23
24/* Radix page table supported and enabled */
25#define MMU_FTR_TYPE_RADIX ASM_CONST(0x00000040)
26
27/*
28 * Individual features below.
29 */
30
31/*
32 * Supports KUAP feature
33 * key 0 controlling userspace addresses on radix
34 * Key 3 on hash
35 */
36#define MMU_FTR_KUAP ASM_CONST(0x00000200)
37
38/*
39 * Supports KUEP feature
40 * key 0 controlling userspace addresses on radix
41 * Key 3 on hash
42 */
43#define MMU_FTR_BOOK3S_KUEP ASM_CONST(0x00000400)
44
45/*
46 * Support for memory protection keys.
47 */
48#define MMU_FTR_PKEY ASM_CONST(0x00000800)
49
50/* Guest Translation Shootdown Enable */
51#define MMU_FTR_GTSE ASM_CONST(0x00001000)
52
53/*
54 * Support for 68 bit VA space. We added that from ISA 2.05
55 */
56#define MMU_FTR_68_BIT_VA ASM_CONST(0x00002000)
57/*
58 * Kernel read only support.
59 * We added the ppp value 0b110 in ISA 2.04.
60 */
61#define MMU_FTR_KERNEL_RO ASM_CONST(0x00004000)
62
63/*
64 * We need to clear top 16bits of va (from the remaining 64 bits )in
65 * tlbie* instructions
66 */
67#define MMU_FTR_TLBIE_CROP_VA ASM_CONST(0x00008000)
68
69/* Enable use of high BAT registers */
70#define MMU_FTR_USE_HIGH_BATS ASM_CONST(0x00010000)
71
72/* Enable >32-bit physical addresses on 32-bit processor, only used
73 * by CONFIG_PPC_BOOK3S_32 currently as BookE supports that from day 1
74 */
75#define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000)
76
77/* Enable use of broadcast TLB invalidations. We don't always set it
78 * on processors that support it due to other constraints with the
79 * use of such invalidations
80 */
81#define MMU_FTR_USE_TLBIVAX_BCAST ASM_CONST(0x00040000)
82
83/* Enable use of tlbilx invalidate instructions.
84 */
85#define MMU_FTR_USE_TLBILX ASM_CONST(0x00080000)
86
87/* This indicates that the processor cannot handle multiple outstanding
88 * broadcast tlbivax or tlbsync. This makes the code use a spinlock
89 * around such invalidate forms.
90 */
91#define MMU_FTR_LOCK_BCAST_INVAL ASM_CONST(0x00100000)
92
93/* This indicates that the processor doesn't handle way selection
94 * properly and needs SW to track and update the LRU state. This
95 * is specific to an errata on e300c2/c3/c4 class parts
96 */
97#define MMU_FTR_NEED_DTLB_SW_LRU ASM_CONST(0x00200000)
98
99/* Doesn't support the B bit (1T segment) in SLBIE
100 */
101#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x02000000)
102
103/* Support 16M large pages
104 */
105#define MMU_FTR_16M_PAGE ASM_CONST(0x04000000)
106
107/* Supports TLBIEL variant
108 */
109#define MMU_FTR_TLBIEL ASM_CONST(0x08000000)
110
111/* Supports tlbies w/o locking
112 */
113#define MMU_FTR_LOCKLESS_TLBIE ASM_CONST(0x10000000)
114
115/* Large pages can be marked CI
116 */
117#define MMU_FTR_CI_LARGE_PAGE ASM_CONST(0x20000000)
118
119/* 1T segments available
120 */
121#define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000)
122
123// NX paste RMA reject in DSI
124#define MMU_FTR_NX_DSI ASM_CONST(0x80000000)
125
126/* MMU feature bit sets for various CPUs */
127#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 (MMU_FTR_HPTE_TABLE | MMU_FTR_TLBIEL | MMU_FTR_16M_PAGE)
128#define MMU_FTRS_POWER MMU_FTRS_DEFAULT_HPTE_ARCH_V2
129#define MMU_FTRS_PPC970 MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA
130#define MMU_FTRS_POWER5 MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE
131#define MMU_FTRS_POWER6 MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA
132#define MMU_FTRS_POWER7 MMU_FTRS_POWER6
133#define MMU_FTRS_POWER8 MMU_FTRS_POWER6
134#define MMU_FTRS_POWER9 MMU_FTRS_POWER6
135#define MMU_FTRS_POWER10 MMU_FTRS_POWER6
136#define MMU_FTRS_POWER11 MMU_FTRS_POWER6
137#define MMU_FTRS_CELL MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
138 MMU_FTR_CI_LARGE_PAGE
139#define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
140 MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
141#ifndef __ASSEMBLY__
142#include <linux/bug.h>
143#include <asm/cputable.h>
144#include <asm/page.h>
145
146typedef pte_t *pgtable_t;
147
148enum {
149 MMU_FTRS_POSSIBLE =
150#if defined(CONFIG_PPC_BOOK3S_604)
151 MMU_FTR_HPTE_TABLE |
152#endif
153#ifdef CONFIG_PPC_8xx
154 MMU_FTR_TYPE_8xx |
155#endif
156#ifdef CONFIG_40x
157 MMU_FTR_TYPE_40x |
158#endif
159#ifdef CONFIG_PPC_47x
160 MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL |
161#elif defined(CONFIG_44x)
162 MMU_FTR_TYPE_44x |
163#endif
164#ifdef CONFIG_PPC_E500
165 MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX |
166#endif
167#ifdef CONFIG_PPC_BOOK3S_32
168 MMU_FTR_USE_HIGH_BATS |
169#endif
170#ifdef CONFIG_PPC_83xx
171 MMU_FTR_NEED_DTLB_SW_LRU |
172#endif
173#ifdef CONFIG_PPC_BOOK3S_64
174 MMU_FTR_KERNEL_RO |
175#ifdef CONFIG_PPC_64S_HASH_MMU
176 MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
177 MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
178 MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
179 MMU_FTR_68_BIT_VA | MMU_FTR_HPTE_TABLE |
180#endif
181#ifdef CONFIG_PPC_RADIX_MMU
182 MMU_FTR_TYPE_RADIX |
183 MMU_FTR_GTSE | MMU_FTR_NX_DSI |
184#endif /* CONFIG_PPC_RADIX_MMU */
185#endif
186#ifdef CONFIG_PPC_KUAP
187 MMU_FTR_KUAP |
188#endif /* CONFIG_PPC_KUAP */
189#ifdef CONFIG_PPC_MEM_KEYS
190 MMU_FTR_PKEY |
191#endif
192#ifdef CONFIG_PPC_KUEP
193 MMU_FTR_BOOK3S_KUEP |
194#endif /* CONFIG_PPC_KUAP */
195
196 0,
197};
198
199#if defined(CONFIG_PPC_BOOK3S_604) && !defined(CONFIG_PPC_BOOK3S_603)
200#define MMU_FTRS_ALWAYS MMU_FTR_HPTE_TABLE
201#endif
202#ifdef CONFIG_PPC_8xx
203#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_8xx
204#endif
205#ifdef CONFIG_40x
206#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_40x
207#endif
208#ifdef CONFIG_PPC_47x
209#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_47x
210#elif defined(CONFIG_44x)
211#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_44x
212#endif
213#ifdef CONFIG_PPC_E500
214#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_FSL_E
215#endif
216
217/* BOOK3S_64 options */
218#if defined(CONFIG_PPC_RADIX_MMU) && !defined(CONFIG_PPC_64S_HASH_MMU)
219#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_RADIX
220#elif !defined(CONFIG_PPC_RADIX_MMU) && defined(CONFIG_PPC_64S_HASH_MMU)
221#define MMU_FTRS_ALWAYS MMU_FTR_HPTE_TABLE
222#endif
223
224#ifndef MMU_FTRS_ALWAYS
225#define MMU_FTRS_ALWAYS 0
226#endif
227
228static __always_inline bool early_mmu_has_feature(unsigned long feature)
229{
230 if (MMU_FTRS_ALWAYS & feature)
231 return true;
232
233 return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
234}
235
236#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
237#include <linux/jump_label.h>
238
239#define NUM_MMU_FTR_KEYS 32
240
241extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
242
243extern void mmu_feature_keys_init(void);
244
245static __always_inline bool mmu_has_feature(unsigned long feature)
246{
247 int i;
248
249#ifndef __clang__ /* clang can't cope with this */
250 BUILD_BUG_ON(!__builtin_constant_p(feature));
251#endif
252
253#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
254 if (!static_key_initialized) {
255 printk("Warning! mmu_has_feature() used prior to jump label init!\n");
256 dump_stack();
257 return early_mmu_has_feature(feature);
258 }
259#endif
260
261 if (MMU_FTRS_ALWAYS & feature)
262 return true;
263
264 if (!(MMU_FTRS_POSSIBLE & feature))
265 return false;
266
267 i = __builtin_ctzl(feature);
268 return static_branch_likely(&mmu_feature_keys[i]);
269}
270
271static inline void mmu_clear_feature(unsigned long feature)
272{
273 int i;
274
275 i = __builtin_ctzl(feature);
276 cur_cpu_spec->mmu_features &= ~feature;
277 static_branch_disable(&mmu_feature_keys[i]);
278}
279#else
280
281static inline void mmu_feature_keys_init(void)
282{
283
284}
285
286static __always_inline bool mmu_has_feature(unsigned long feature)
287{
288 return early_mmu_has_feature(feature);
289}
290
291static inline void mmu_clear_feature(unsigned long feature)
292{
293 cur_cpu_spec->mmu_features &= ~feature;
294}
295#endif /* CONFIG_JUMP_LABEL */
296
297extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
298
299#ifdef CONFIG_PPC64
300/* This is our real memory area size on ppc64 server, on embedded, we
301 * make it match the size our of bolted TLB area
302 */
303extern u64 ppc64_rma_size;
304
305/* Cleanup function used by kexec */
306extern void mmu_cleanup_all(void);
307extern void radix__mmu_cleanup_all(void);
308
309/* Functions for creating and updating partition table on POWER9 */
310extern void mmu_partition_table_init(void);
311extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
312 unsigned long dw1, bool flush);
313#endif /* CONFIG_PPC64 */
314
315struct mm_struct;
316#ifdef CONFIG_DEBUG_VM
317extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
318#else /* CONFIG_DEBUG_VM */
319static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
320{
321}
322#endif /* !CONFIG_DEBUG_VM */
323
324static __always_inline bool radix_enabled(void)
325{
326 return mmu_has_feature(MMU_FTR_TYPE_RADIX);
327}
328
329static __always_inline bool early_radix_enabled(void)
330{
331 return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
332}
333
334static inline bool strict_kernel_rwx_enabled(void)
335{
336 return IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled;
337}
338
339static inline bool strict_module_rwx_enabled(void)
340{
341 return IS_ENABLED(CONFIG_STRICT_MODULE_RWX) && strict_kernel_rwx_enabled();
342}
343#endif /* !__ASSEMBLY__ */
344
345/* The kernel use the constants below to index in the page sizes array.
346 * The use of fixed constants for this purpose is better for performances
347 * of the low level hash refill handlers.
348 *
349 * A non supported page size has a "shift" field set to 0
350 *
351 * Any new page size being implemented can get a new entry in here. Whether
352 * the kernel will use it or not is a different matter though. The actual page
353 * size used by hugetlbfs is not defined here and may be made variable
354 *
355 * Note: This array ended up being a false good idea as it's growing to the
356 * point where I wonder if we should replace it with something different,
357 * to think about, feedback welcome. --BenH.
358 */
359
360/* These are #defines as they have to be used in assembly */
361#define MMU_PAGE_4K 0
362#define MMU_PAGE_16K 1
363#define MMU_PAGE_64K 2
364#define MMU_PAGE_64K_AP 3 /* "Admixed pages" (hash64 only) */
365#define MMU_PAGE_256K 4
366#define MMU_PAGE_512K 5
367#define MMU_PAGE_1M 6
368#define MMU_PAGE_2M 7
369#define MMU_PAGE_4M 8
370#define MMU_PAGE_8M 9
371#define MMU_PAGE_16M 10
372#define MMU_PAGE_64M 11
373#define MMU_PAGE_256M 12
374#define MMU_PAGE_1G 13
375#define MMU_PAGE_16G 14
376#define MMU_PAGE_64G 15
377
378/*
379 * N.B. we need to change the type of hpte_page_sizes if this gets to be > 16
380 * Also we need to change he type of mm_context.low/high_slices_psize.
381 */
382#define MMU_PAGE_COUNT 16
383
384#ifdef CONFIG_PPC_BOOK3S_64
385#include <asm/book3s/64/mmu.h>
386#else /* CONFIG_PPC_BOOK3S_64 */
387
388#ifndef __ASSEMBLY__
389/* MMU initialization */
390extern void early_init_mmu(void);
391extern void early_init_mmu_secondary(void);
392extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
393 phys_addr_t first_memblock_size);
394static inline void mmu_early_init_devtree(void) { }
395
396static inline void pkey_early_init_devtree(void) {}
397
398extern void *abatron_pteptrs[2];
399#endif /* __ASSEMBLY__ */
400#endif
401
402#if defined(CONFIG_PPC_BOOK3S_32)
403/* 32-bit classic hash table MMU */
404#include <asm/book3s/32/mmu-hash.h>
405#elif defined(CONFIG_PPC_MMU_NOHASH)
406#include <asm/nohash/mmu.h>
407#endif
408
409#if defined(CONFIG_FA_DUMP) || defined(CONFIG_PRESERVE_FA_DUMP)
410#define __HAVE_ARCH_RESERVED_KERNEL_PAGES
411#endif
412
413#endif /* __KERNEL__ */
414#endif /* _ASM_POWERPC_MMU_H_ */
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_POWERPC_MMU_H_
3#define _ASM_POWERPC_MMU_H_
4#ifdef __KERNEL__
5
6#include <linux/types.h>
7
8#include <asm/asm-compat.h>
9#include <asm/feature-fixups.h>
10
11/*
12 * MMU features bit definitions
13 */
14
15/*
16 * MMU families
17 */
18#define MMU_FTR_HPTE_TABLE ASM_CONST(0x00000001)
19#define MMU_FTR_TYPE_8xx ASM_CONST(0x00000002)
20#define MMU_FTR_TYPE_40x ASM_CONST(0x00000004)
21#define MMU_FTR_TYPE_44x ASM_CONST(0x00000008)
22#define MMU_FTR_TYPE_FSL_E ASM_CONST(0x00000010)
23#define MMU_FTR_TYPE_47x ASM_CONST(0x00000020)
24
25/* Radix page table supported and enabled */
26#define MMU_FTR_TYPE_RADIX ASM_CONST(0x00000040)
27
28/*
29 * Individual features below.
30 */
31
32/*
33 * Support for 68 bit VA space. We added that from ISA 2.05
34 */
35#define MMU_FTR_68_BIT_VA ASM_CONST(0x00002000)
36/*
37 * Kernel read only support.
38 * We added the ppp value 0b110 in ISA 2.04.
39 */
40#define MMU_FTR_KERNEL_RO ASM_CONST(0x00004000)
41
42/*
43 * We need to clear top 16bits of va (from the remaining 64 bits )in
44 * tlbie* instructions
45 */
46#define MMU_FTR_TLBIE_CROP_VA ASM_CONST(0x00008000)
47
48/* Enable use of high BAT registers */
49#define MMU_FTR_USE_HIGH_BATS ASM_CONST(0x00010000)
50
51/* Enable >32-bit physical addresses on 32-bit processor, only used
52 * by CONFIG_6xx currently as BookE supports that from day 1
53 */
54#define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000)
55
56/* Enable use of broadcast TLB invalidations. We don't always set it
57 * on processors that support it due to other constraints with the
58 * use of such invalidations
59 */
60#define MMU_FTR_USE_TLBIVAX_BCAST ASM_CONST(0x00040000)
61
62/* Enable use of tlbilx invalidate instructions.
63 */
64#define MMU_FTR_USE_TLBILX ASM_CONST(0x00080000)
65
66/* This indicates that the processor cannot handle multiple outstanding
67 * broadcast tlbivax or tlbsync. This makes the code use a spinlock
68 * around such invalidate forms.
69 */
70#define MMU_FTR_LOCK_BCAST_INVAL ASM_CONST(0x00100000)
71
72/* This indicates that the processor doesn't handle way selection
73 * properly and needs SW to track and update the LRU state. This
74 * is specific to an errata on e300c2/c3/c4 class parts
75 */
76#define MMU_FTR_NEED_DTLB_SW_LRU ASM_CONST(0x00200000)
77
78/* Enable use of TLB reservation. Processor should support tlbsrx.
79 * instruction and MAS0[WQ].
80 */
81#define MMU_FTR_USE_TLBRSRV ASM_CONST(0x00800000)
82
83/* Use paired MAS registers (MAS7||MAS3, etc.)
84 */
85#define MMU_FTR_USE_PAIRED_MAS ASM_CONST(0x01000000)
86
87/* Doesn't support the B bit (1T segment) in SLBIE
88 */
89#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x02000000)
90
91/* Support 16M large pages
92 */
93#define MMU_FTR_16M_PAGE ASM_CONST(0x04000000)
94
95/* Supports TLBIEL variant
96 */
97#define MMU_FTR_TLBIEL ASM_CONST(0x08000000)
98
99/* Supports tlbies w/o locking
100 */
101#define MMU_FTR_LOCKLESS_TLBIE ASM_CONST(0x10000000)
102
103/* Large pages can be marked CI
104 */
105#define MMU_FTR_CI_LARGE_PAGE ASM_CONST(0x20000000)
106
107/* 1T segments available
108 */
109#define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000)
110
111/* MMU feature bit sets for various CPUs */
112#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 \
113 MMU_FTR_HPTE_TABLE | MMU_FTR_PPCAS_ARCH_V2
114#define MMU_FTRS_POWER MMU_FTRS_DEFAULT_HPTE_ARCH_V2
115#define MMU_FTRS_PPC970 MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA
116#define MMU_FTRS_POWER5 MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE
117#define MMU_FTRS_POWER6 MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA
118#define MMU_FTRS_POWER7 MMU_FTRS_POWER6
119#define MMU_FTRS_POWER8 MMU_FTRS_POWER6
120#define MMU_FTRS_POWER9 MMU_FTRS_POWER6
121#define MMU_FTRS_CELL MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
122 MMU_FTR_CI_LARGE_PAGE
123#define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
124 MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
125#ifndef __ASSEMBLY__
126#include <linux/bug.h>
127#include <asm/cputable.h>
128
129#ifdef CONFIG_PPC_FSL_BOOK3E
130#include <asm/percpu.h>
131DECLARE_PER_CPU(int, next_tlbcam_idx);
132#endif
133
134enum {
135 MMU_FTRS_POSSIBLE = MMU_FTR_HPTE_TABLE | MMU_FTR_TYPE_8xx |
136 MMU_FTR_TYPE_40x | MMU_FTR_TYPE_44x | MMU_FTR_TYPE_FSL_E |
137 MMU_FTR_TYPE_47x | MMU_FTR_USE_HIGH_BATS | MMU_FTR_BIG_PHYS |
138 MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_USE_TLBILX |
139 MMU_FTR_LOCK_BCAST_INVAL | MMU_FTR_NEED_DTLB_SW_LRU |
140 MMU_FTR_USE_TLBRSRV | MMU_FTR_USE_PAIRED_MAS |
141 MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
142 MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
143 MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
144 MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA |
145#ifdef CONFIG_PPC_RADIX_MMU
146 MMU_FTR_TYPE_RADIX |
147#endif
148 0,
149};
150
151static inline bool early_mmu_has_feature(unsigned long feature)
152{
153 return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
154}
155
156#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
157#include <linux/jump_label.h>
158
159#define NUM_MMU_FTR_KEYS 32
160
161extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
162
163extern void mmu_feature_keys_init(void);
164
165static __always_inline bool mmu_has_feature(unsigned long feature)
166{
167 int i;
168
169#ifndef __clang__ /* clang can't cope with this */
170 BUILD_BUG_ON(!__builtin_constant_p(feature));
171#endif
172
173#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
174 if (!static_key_initialized) {
175 printk("Warning! mmu_has_feature() used prior to jump label init!\n");
176 dump_stack();
177 return early_mmu_has_feature(feature);
178 }
179#endif
180
181 if (!(MMU_FTRS_POSSIBLE & feature))
182 return false;
183
184 i = __builtin_ctzl(feature);
185 return static_branch_likely(&mmu_feature_keys[i]);
186}
187
188static inline void mmu_clear_feature(unsigned long feature)
189{
190 int i;
191
192 i = __builtin_ctzl(feature);
193 cur_cpu_spec->mmu_features &= ~feature;
194 static_branch_disable(&mmu_feature_keys[i]);
195}
196#else
197
198static inline void mmu_feature_keys_init(void)
199{
200
201}
202
203static inline bool mmu_has_feature(unsigned long feature)
204{
205 return early_mmu_has_feature(feature);
206}
207
208static inline void mmu_clear_feature(unsigned long feature)
209{
210 cur_cpu_spec->mmu_features &= ~feature;
211}
212#endif /* CONFIG_JUMP_LABEL */
213
214extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
215
216#ifdef CONFIG_PPC64
217/* This is our real memory area size on ppc64 server, on embedded, we
218 * make it match the size our of bolted TLB area
219 */
220extern u64 ppc64_rma_size;
221
222/* Cleanup function used by kexec */
223extern void mmu_cleanup_all(void);
224extern void radix__mmu_cleanup_all(void);
225
226/* Functions for creating and updating partition table on POWER9 */
227extern void mmu_partition_table_init(void);
228extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
229 unsigned long dw1);
230#endif /* CONFIG_PPC64 */
231
232struct mm_struct;
233#ifdef CONFIG_DEBUG_VM
234extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
235#else /* CONFIG_DEBUG_VM */
236static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
237{
238}
239#endif /* !CONFIG_DEBUG_VM */
240
241#ifdef CONFIG_PPC_RADIX_MMU
242static inline bool radix_enabled(void)
243{
244 return mmu_has_feature(MMU_FTR_TYPE_RADIX);
245}
246
247static inline bool early_radix_enabled(void)
248{
249 return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
250}
251#else
252static inline bool radix_enabled(void)
253{
254 return false;
255}
256
257static inline bool early_radix_enabled(void)
258{
259 return false;
260}
261#endif
262
263#ifdef CONFIG_PPC_MEM_KEYS
264extern u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address);
265#else
266static inline u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address)
267{
268 return 0;
269}
270#endif /* CONFIG_PPC_MEM_KEYS */
271
272#endif /* !__ASSEMBLY__ */
273
274/* The kernel use the constants below to index in the page sizes array.
275 * The use of fixed constants for this purpose is better for performances
276 * of the low level hash refill handlers.
277 *
278 * A non supported page size has a "shift" field set to 0
279 *
280 * Any new page size being implemented can get a new entry in here. Whether
281 * the kernel will use it or not is a different matter though. The actual page
282 * size used by hugetlbfs is not defined here and may be made variable
283 *
284 * Note: This array ended up being a false good idea as it's growing to the
285 * point where I wonder if we should replace it with something different,
286 * to think about, feedback welcome. --BenH.
287 */
288
289/* These are #defines as they have to be used in assembly */
290#define MMU_PAGE_4K 0
291#define MMU_PAGE_16K 1
292#define MMU_PAGE_64K 2
293#define MMU_PAGE_64K_AP 3 /* "Admixed pages" (hash64 only) */
294#define MMU_PAGE_256K 4
295#define MMU_PAGE_512K 5
296#define MMU_PAGE_1M 6
297#define MMU_PAGE_2M 7
298#define MMU_PAGE_4M 8
299#define MMU_PAGE_8M 9
300#define MMU_PAGE_16M 10
301#define MMU_PAGE_64M 11
302#define MMU_PAGE_256M 12
303#define MMU_PAGE_1G 13
304#define MMU_PAGE_16G 14
305#define MMU_PAGE_64G 15
306
307/*
308 * N.B. we need to change the type of hpte_page_sizes if this gets to be > 16
309 * Also we need to change he type of mm_context.low/high_slices_psize.
310 */
311#define MMU_PAGE_COUNT 16
312
313#ifdef CONFIG_PPC_BOOK3S_64
314#include <asm/book3s/64/mmu.h>
315#else /* CONFIG_PPC_BOOK3S_64 */
316
317#ifndef __ASSEMBLY__
318/* MMU initialization */
319extern void early_init_mmu(void);
320extern void early_init_mmu_secondary(void);
321extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
322 phys_addr_t first_memblock_size);
323static inline void mmu_early_init_devtree(void) { }
324#endif /* __ASSEMBLY__ */
325#endif
326
327#if defined(CONFIG_PPC_STD_MMU_32)
328/* 32-bit classic hash table MMU */
329#include <asm/book3s/32/mmu-hash.h>
330#elif defined(CONFIG_40x)
331/* 40x-style software loaded TLB */
332# include <asm/mmu-40x.h>
333#elif defined(CONFIG_44x)
334/* 44x-style software loaded TLB */
335# include <asm/mmu-44x.h>
336#elif defined(CONFIG_PPC_BOOK3E_MMU)
337/* Freescale Book-E software loaded TLB or Book-3e (ISA 2.06+) MMU */
338# include <asm/mmu-book3e.h>
339#elif defined (CONFIG_PPC_8xx)
340/* Motorola/Freescale 8xx software loaded TLB */
341# include <asm/mmu-8xx.h>
342#endif
343
344#endif /* __KERNEL__ */
345#endif /* _ASM_POWERPC_MMU_H_ */