Loading...
Note: File does not exist in v6.8.
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PowerPC Memory Protection Keys management
4 *
5 * Copyright 2017, Ram Pai, IBM Corporation.
6 */
7
8#include <asm/mman.h>
9#include <asm/setup.h>
10#include <linux/pkeys.h>
11#include <linux/of_device.h>
12
13DEFINE_STATIC_KEY_TRUE(pkey_disabled);
14bool pkey_execute_disable_supported;
15int pkeys_total; /* Total pkeys as per device tree */
16bool pkeys_devtree_defined; /* pkey property exported by device tree */
17u32 initial_allocation_mask; /* Bits set for reserved keys */
18u64 pkey_amr_uamor_mask; /* Bits in AMR/UMOR not to be touched */
19u64 pkey_iamr_mask; /* Bits in AMR not to be touched */
20
21#define AMR_BITS_PER_PKEY 2
22#define AMR_RD_BIT 0x1UL
23#define AMR_WR_BIT 0x2UL
24#define IAMR_EX_BIT 0x1UL
25#define PKEY_REG_BITS (sizeof(u64)*8)
26#define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
27
28static void scan_pkey_feature(void)
29{
30 u32 vals[2];
31 struct device_node *cpu;
32
33 cpu = of_find_node_by_type(NULL, "cpu");
34 if (!cpu)
35 return;
36
37 if (of_property_read_u32_array(cpu,
38 "ibm,processor-storage-keys", vals, 2))
39 return;
40
41 /*
42 * Since any pkey can be used for data or execute, we will just treat
43 * all keys as equal and track them as one entity.
44 */
45 pkeys_total = be32_to_cpu(vals[0]);
46 pkeys_devtree_defined = true;
47}
48
49static inline bool pkey_mmu_enabled(void)
50{
51 if (firmware_has_feature(FW_FEATURE_LPAR))
52 return pkeys_total;
53 else
54 return cpu_has_feature(CPU_FTR_PKEY);
55}
56
57int pkey_initialize(void)
58{
59 int os_reserved, i;
60
61 /*
62 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
63 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE.
64 * Ensure that the bits a distinct.
65 */
66 BUILD_BUG_ON(PKEY_DISABLE_EXECUTE &
67 (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
68
69 /*
70 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous
71 * in the vmaflag. Make sure that is really the case.
72 */
73 BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) +
74 __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)
75 != (sizeof(u64) * BITS_PER_BYTE));
76
77 /* scan the device tree for pkey feature */
78 scan_pkey_feature();
79
80 /*
81 * Let's assume 32 pkeys on P8 bare metal, if its not defined by device
82 * tree. We make this exception since skiboot forgot to expose this
83 * property on power8.
84 */
85 if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) &&
86 cpu_has_feature(CPU_FTRS_POWER8))
87 pkeys_total = 32;
88
89 /*
90 * Adjust the upper limit, based on the number of bits supported by
91 * arch-neutral code.
92 */
93 pkeys_total = min_t(int, pkeys_total,
94 (ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT));
95
96 if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total)
97 static_branch_enable(&pkey_disabled);
98 else
99 static_branch_disable(&pkey_disabled);
100
101 if (static_branch_likely(&pkey_disabled))
102 return 0;
103
104 /*
105 * The device tree cannot be relied to indicate support for
106 * execute_disable support. Instead we use a PVR check.
107 */
108 if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p))
109 pkey_execute_disable_supported = false;
110 else
111 pkey_execute_disable_supported = true;
112
113#ifdef CONFIG_PPC_4K_PAGES
114 /*
115 * The OS can manage only 8 pkeys due to its inability to represent them
116 * in the Linux 4K PTE.
117 */
118 os_reserved = pkeys_total - 8;
119#else
120 os_reserved = 0;
121#endif
122 initial_allocation_mask = ~0x0;
123 pkey_amr_uamor_mask = ~0x0ul;
124 pkey_iamr_mask = ~0x0ul;
125 /*
126 * key 0, 1 are reserved.
127 * key 0 is the default key, which allows read/write/execute.
128 * key 1 is recommended not to be used. PowerISA(3.0) page 1015,
129 * programming note.
130 */
131 for (i = 2; i < (pkeys_total - os_reserved); i++) {
132 initial_allocation_mask &= ~(0x1 << i);
133 pkey_amr_uamor_mask &= ~(0x3ul << pkeyshift(i));
134 pkey_iamr_mask &= ~(0x1ul << pkeyshift(i));
135 }
136 return 0;
137}
138
139arch_initcall(pkey_initialize);
140
141void pkey_mm_init(struct mm_struct *mm)
142{
143 if (static_branch_likely(&pkey_disabled))
144 return;
145 mm_pkey_allocation_map(mm) = initial_allocation_mask;
146 /* -1 means unallocated or invalid */
147 mm->context.execute_only_pkey = -1;
148}
149
150static inline u64 read_amr(void)
151{
152 return mfspr(SPRN_AMR);
153}
154
155static inline void write_amr(u64 value)
156{
157 mtspr(SPRN_AMR, value);
158}
159
160static inline u64 read_iamr(void)
161{
162 if (!likely(pkey_execute_disable_supported))
163 return 0x0UL;
164
165 return mfspr(SPRN_IAMR);
166}
167
168static inline void write_iamr(u64 value)
169{
170 if (!likely(pkey_execute_disable_supported))
171 return;
172
173 mtspr(SPRN_IAMR, value);
174}
175
176static inline u64 read_uamor(void)
177{
178 return mfspr(SPRN_UAMOR);
179}
180
181static inline void write_uamor(u64 value)
182{
183 mtspr(SPRN_UAMOR, value);
184}
185
186static bool is_pkey_enabled(int pkey)
187{
188 u64 uamor = read_uamor();
189 u64 pkey_bits = 0x3ul << pkeyshift(pkey);
190 u64 uamor_pkey_bits = (uamor & pkey_bits);
191
192 /*
193 * Both the bits in UAMOR corresponding to the key should be set or
194 * reset.
195 */
196 WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits));
197 return !!(uamor_pkey_bits);
198}
199
200static inline void init_amr(int pkey, u8 init_bits)
201{
202 u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
203 u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
204
205 write_amr(old_amr | new_amr_bits);
206}
207
208static inline void init_iamr(int pkey, u8 init_bits)
209{
210 u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
211 u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
212
213 write_iamr(old_iamr | new_iamr_bits);
214}
215
216static void pkey_status_change(int pkey, bool enable)
217{
218 u64 old_uamor;
219
220 /* Reset the AMR and IAMR bits for this key */
221 init_amr(pkey, 0x0);
222 init_iamr(pkey, 0x0);
223
224 /* Enable/disable key */
225 old_uamor = read_uamor();
226 if (enable)
227 old_uamor |= (0x3ul << pkeyshift(pkey));
228 else
229 old_uamor &= ~(0x3ul << pkeyshift(pkey));
230 write_uamor(old_uamor);
231}
232
233void __arch_activate_pkey(int pkey)
234{
235 pkey_status_change(pkey, true);
236}
237
238void __arch_deactivate_pkey(int pkey)
239{
240 pkey_status_change(pkey, false);
241}
242
243/*
244 * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
245 * specified in @init_val.
246 */
247int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
248 unsigned long init_val)
249{
250 u64 new_amr_bits = 0x0ul;
251 u64 new_iamr_bits = 0x0ul;
252
253 if (!is_pkey_enabled(pkey))
254 return -EINVAL;
255
256 if (init_val & PKEY_DISABLE_EXECUTE) {
257 if (!pkey_execute_disable_supported)
258 return -EINVAL;
259 new_iamr_bits |= IAMR_EX_BIT;
260 }
261 init_iamr(pkey, new_iamr_bits);
262
263 /* Set the bits we need in AMR: */
264 if (init_val & PKEY_DISABLE_ACCESS)
265 new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT;
266 else if (init_val & PKEY_DISABLE_WRITE)
267 new_amr_bits |= AMR_WR_BIT;
268
269 init_amr(pkey, new_amr_bits);
270 return 0;
271}
272
273void thread_pkey_regs_save(struct thread_struct *thread)
274{
275 if (static_branch_likely(&pkey_disabled))
276 return;
277
278 /*
279 * TODO: Skip saving registers if @thread hasn't used any keys yet.
280 */
281 thread->amr = read_amr();
282 thread->iamr = read_iamr();
283 thread->uamor = read_uamor();
284}
285
286void thread_pkey_regs_restore(struct thread_struct *new_thread,
287 struct thread_struct *old_thread)
288{
289 if (static_branch_likely(&pkey_disabled))
290 return;
291
292 /*
293 * TODO: Just set UAMOR to zero if @new_thread hasn't used any keys yet.
294 */
295 if (old_thread->amr != new_thread->amr)
296 write_amr(new_thread->amr);
297 if (old_thread->iamr != new_thread->iamr)
298 write_iamr(new_thread->iamr);
299 if (old_thread->uamor != new_thread->uamor)
300 write_uamor(new_thread->uamor);
301}
302
303void thread_pkey_regs_init(struct thread_struct *thread)
304{
305 if (static_branch_likely(&pkey_disabled))
306 return;
307
308 thread->amr = read_amr() & pkey_amr_uamor_mask;
309 thread->iamr = read_iamr() & pkey_iamr_mask;
310 thread->uamor = read_uamor() & pkey_amr_uamor_mask;
311}
312
313static inline bool pkey_allows_readwrite(int pkey)
314{
315 int pkey_shift = pkeyshift(pkey);
316
317 if (!is_pkey_enabled(pkey))
318 return true;
319
320 return !(read_amr() & ((AMR_RD_BIT|AMR_WR_BIT) << pkey_shift));
321}
322
323int __execute_only_pkey(struct mm_struct *mm)
324{
325 bool need_to_set_mm_pkey = false;
326 int execute_only_pkey = mm->context.execute_only_pkey;
327 int ret;
328
329 /* Do we need to assign a pkey for mm's execute-only maps? */
330 if (execute_only_pkey == -1) {
331 /* Go allocate one to use, which might fail */
332 execute_only_pkey = mm_pkey_alloc(mm);
333 if (execute_only_pkey < 0)
334 return -1;
335 need_to_set_mm_pkey = true;
336 }
337
338 /*
339 * We do not want to go through the relatively costly dance to set AMR
340 * if we do not need to. Check it first and assume that if the
341 * execute-only pkey is readwrite-disabled than we do not have to set it
342 * ourselves.
343 */
344 if (!need_to_set_mm_pkey && !pkey_allows_readwrite(execute_only_pkey))
345 return execute_only_pkey;
346
347 /*
348 * Set up AMR so that it denies access for everything other than
349 * execution.
350 */
351 ret = __arch_set_user_pkey_access(current, execute_only_pkey,
352 PKEY_DISABLE_ACCESS |
353 PKEY_DISABLE_WRITE);
354 /*
355 * If the AMR-set operation failed somehow, just return 0 and
356 * effectively disable execute-only support.
357 */
358 if (ret) {
359 mm_pkey_free(mm, execute_only_pkey);
360 return -1;
361 }
362
363 /* We got one, store it and use it from here on out */
364 if (need_to_set_mm_pkey)
365 mm->context.execute_only_pkey = execute_only_pkey;
366 return execute_only_pkey;
367}
368
369static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma)
370{
371 /* Do this check first since the vm_flags should be hot */
372 if ((vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) != VM_EXEC)
373 return false;
374
375 return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey);
376}
377
378/*
379 * This should only be called for *plain* mprotect calls.
380 */
381int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot,
382 int pkey)
383{
384 /*
385 * If the currently associated pkey is execute-only, but the requested
386 * protection requires read or write, move it back to the default pkey.
387 */
388 if (vma_is_pkey_exec_only(vma) && (prot & (PROT_READ | PROT_WRITE)))
389 return 0;
390
391 /*
392 * The requested protection is execute-only. Hence let's use an
393 * execute-only pkey.
394 */
395 if (prot == PROT_EXEC) {
396 pkey = execute_only_pkey(vma->vm_mm);
397 if (pkey > 0)
398 return pkey;
399 }
400
401 /* Nothing to override. */
402 return vma_pkey(vma);
403}
404
405static bool pkey_access_permitted(int pkey, bool write, bool execute)
406{
407 int pkey_shift;
408 u64 amr;
409
410 if (!pkey)
411 return true;
412
413 if (!is_pkey_enabled(pkey))
414 return true;
415
416 pkey_shift = pkeyshift(pkey);
417 if (execute && !(read_iamr() & (IAMR_EX_BIT << pkey_shift)))
418 return true;
419
420 amr = read_amr(); /* Delay reading amr until absolutely needed */
421 return ((!write && !(amr & (AMR_RD_BIT << pkey_shift))) ||
422 (write && !(amr & (AMR_WR_BIT << pkey_shift))));
423}
424
425bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
426{
427 if (static_branch_likely(&pkey_disabled))
428 return true;
429
430 return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute);
431}
432
433/*
434 * We only want to enforce protection keys on the current thread because we
435 * effectively have no access to AMR/IAMR for other threads or any way to tell
436 * which AMR/IAMR in a threaded process we could use.
437 *
438 * So do not enforce things if the VMA is not from the current mm, or if we are
439 * in a kernel thread.
440 */
441static inline bool vma_is_foreign(struct vm_area_struct *vma)
442{
443 if (!current->mm)
444 return true;
445
446 /* if it is not our ->mm, it has to be foreign */
447 if (current->mm != vma->vm_mm)
448 return true;
449
450 return false;
451}
452
453bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
454 bool execute, bool foreign)
455{
456 if (static_branch_likely(&pkey_disabled))
457 return true;
458 /*
459 * Do not enforce our key-permissions on a foreign vma.
460 */
461 if (foreign || vma_is_foreign(vma))
462 return true;
463
464 return pkey_access_permitted(vma_pkey(vma), write, execute);
465}