Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2016-20 Intel Corporation. */
3
4#include <linux/lockdep.h>
5#include <linux/mm.h>
6#include <linux/mman.h>
7#include <linux/shmem_fs.h>
8#include <linux/suspend.h>
9#include <linux/sched/mm.h>
10#include <asm/sgx.h>
11#include "encl.h"
12#include "encls.h"
13#include "sgx.h"
14
15static int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index,
16 struct sgx_backing *backing);
17
18#define PCMDS_PER_PAGE (PAGE_SIZE / sizeof(struct sgx_pcmd))
19/*
20 * 32 PCMD entries share a PCMD page. PCMD_FIRST_MASK is used to
21 * determine the page index associated with the first PCMD entry
22 * within a PCMD page.
23 */
24#define PCMD_FIRST_MASK GENMASK(4, 0)
25
26/**
27 * reclaimer_writing_to_pcmd() - Query if any enclave page associated with
28 * a PCMD page is in process of being reclaimed.
29 * @encl: Enclave to which PCMD page belongs
30 * @start_addr: Address of enclave page using first entry within the PCMD page
31 *
32 * When an enclave page is reclaimed some Paging Crypto MetaData (PCMD) is
33 * stored. The PCMD data of a reclaimed enclave page contains enough
34 * information for the processor to verify the page at the time
35 * it is loaded back into the Enclave Page Cache (EPC).
36 *
37 * The backing storage to which enclave pages are reclaimed is laid out as
38 * follows:
39 * Encrypted enclave pages:SECS page:PCMD pages
40 *
41 * Each PCMD page contains the PCMD metadata of
42 * PAGE_SIZE/sizeof(struct sgx_pcmd) enclave pages.
43 *
44 * A PCMD page can only be truncated if it is (a) empty, and (b) not in the
45 * process of getting data (and thus soon being non-empty). (b) is tested with
46 * a check if an enclave page sharing the PCMD page is in the process of being
47 * reclaimed.
48 *
49 * The reclaimer sets the SGX_ENCL_PAGE_BEING_RECLAIMED flag when it
50 * intends to reclaim that enclave page - it means that the PCMD page
51 * associated with that enclave page is about to get some data and thus
52 * even if the PCMD page is empty, it should not be truncated.
53 *
54 * Context: Enclave mutex (&sgx_encl->lock) must be held.
55 * Return: 1 if the reclaimer is about to write to the PCMD page
56 * 0 if the reclaimer has no intention to write to the PCMD page
57 */
58static int reclaimer_writing_to_pcmd(struct sgx_encl *encl,
59 unsigned long start_addr)
60{
61 int reclaimed = 0;
62 int i;
63
64 /*
65 * PCMD_FIRST_MASK is based on number of PCMD entries within
66 * PCMD page being 32.
67 */
68 BUILD_BUG_ON(PCMDS_PER_PAGE != 32);
69
70 for (i = 0; i < PCMDS_PER_PAGE; i++) {
71 struct sgx_encl_page *entry;
72 unsigned long addr;
73
74 addr = start_addr + i * PAGE_SIZE;
75
76 /*
77 * Stop when reaching the SECS page - it does not
78 * have a page_array entry and its reclaim is
79 * started and completed with enclave mutex held so
80 * it does not use the SGX_ENCL_PAGE_BEING_RECLAIMED
81 * flag.
82 */
83 if (addr == encl->base + encl->size)
84 break;
85
86 entry = xa_load(&encl->page_array, PFN_DOWN(addr));
87 if (!entry)
88 continue;
89
90 /*
91 * VA page slot ID uses same bit as the flag so it is important
92 * to ensure that the page is not already in backing store.
93 */
94 if (entry->epc_page &&
95 (entry->desc & SGX_ENCL_PAGE_BEING_RECLAIMED)) {
96 reclaimed = 1;
97 break;
98 }
99 }
100
101 return reclaimed;
102}
103
104/*
105 * Calculate byte offset of a PCMD struct associated with an enclave page. PCMD's
106 * follow right after the EPC data in the backing storage. In addition to the
107 * visible enclave pages, there's one extra page slot for SECS, before PCMD
108 * structs.
109 */
110static inline pgoff_t sgx_encl_get_backing_page_pcmd_offset(struct sgx_encl *encl,
111 unsigned long page_index)
112{
113 pgoff_t epc_end_off = encl->size + sizeof(struct sgx_secs);
114
115 return epc_end_off + page_index * sizeof(struct sgx_pcmd);
116}
117
118/*
119 * Free a page from the backing storage in the given page index.
120 */
121static inline void sgx_encl_truncate_backing_page(struct sgx_encl *encl, unsigned long page_index)
122{
123 struct inode *inode = file_inode(encl->backing);
124
125 shmem_truncate_range(inode, PFN_PHYS(page_index), PFN_PHYS(page_index) + PAGE_SIZE - 1);
126}
127
128/*
129 * ELDU: Load an EPC page as unblocked. For more info, see "OS Management of EPC
130 * Pages" in the SDM.
131 */
132static int __sgx_encl_eldu(struct sgx_encl_page *encl_page,
133 struct sgx_epc_page *epc_page,
134 struct sgx_epc_page *secs_page)
135{
136 unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK;
137 struct sgx_encl *encl = encl_page->encl;
138 pgoff_t page_index, page_pcmd_off;
139 unsigned long pcmd_first_page;
140 struct sgx_pageinfo pginfo;
141 struct sgx_backing b;
142 bool pcmd_page_empty;
143 u8 *pcmd_page;
144 int ret;
145
146 if (secs_page)
147 page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base);
148 else
149 page_index = PFN_DOWN(encl->size);
150
151 /*
152 * Address of enclave page using the first entry within the PCMD page.
153 */
154 pcmd_first_page = PFN_PHYS(page_index & ~PCMD_FIRST_MASK) + encl->base;
155
156 page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index);
157
158 ret = sgx_encl_lookup_backing(encl, page_index, &b);
159 if (ret)
160 return ret;
161
162 pginfo.addr = encl_page->desc & PAGE_MASK;
163 pginfo.contents = (unsigned long)kmap_local_page(b.contents);
164 pcmd_page = kmap_local_page(b.pcmd);
165 pginfo.metadata = (unsigned long)pcmd_page + b.pcmd_offset;
166
167 if (secs_page)
168 pginfo.secs = (u64)sgx_get_epc_virt_addr(secs_page);
169 else
170 pginfo.secs = 0;
171
172 ret = __eldu(&pginfo, sgx_get_epc_virt_addr(epc_page),
173 sgx_get_epc_virt_addr(encl_page->va_page->epc_page) + va_offset);
174 if (ret) {
175 if (encls_failed(ret))
176 ENCLS_WARN(ret, "ELDU");
177
178 ret = -EFAULT;
179 }
180
181 memset(pcmd_page + b.pcmd_offset, 0, sizeof(struct sgx_pcmd));
182 set_page_dirty(b.pcmd);
183
184 /*
185 * The area for the PCMD in the page was zeroed above. Check if the
186 * whole page is now empty meaning that all PCMD's have been zeroed:
187 */
188 pcmd_page_empty = !memchr_inv(pcmd_page, 0, PAGE_SIZE);
189
190 kunmap_local(pcmd_page);
191 kunmap_local((void *)(unsigned long)pginfo.contents);
192
193 get_page(b.pcmd);
194 sgx_encl_put_backing(&b);
195
196 sgx_encl_truncate_backing_page(encl, page_index);
197
198 if (pcmd_page_empty && !reclaimer_writing_to_pcmd(encl, pcmd_first_page)) {
199 sgx_encl_truncate_backing_page(encl, PFN_DOWN(page_pcmd_off));
200 pcmd_page = kmap_local_page(b.pcmd);
201 if (memchr_inv(pcmd_page, 0, PAGE_SIZE))
202 pr_warn("PCMD page not empty after truncate.\n");
203 kunmap_local(pcmd_page);
204 }
205
206 put_page(b.pcmd);
207
208 return ret;
209}
210
211static struct sgx_epc_page *sgx_encl_eldu(struct sgx_encl_page *encl_page,
212 struct sgx_epc_page *secs_page)
213{
214
215 unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK;
216 struct sgx_encl *encl = encl_page->encl;
217 struct sgx_epc_page *epc_page;
218 int ret;
219
220 epc_page = sgx_alloc_epc_page(encl_page, false);
221 if (IS_ERR(epc_page))
222 return epc_page;
223
224 ret = __sgx_encl_eldu(encl_page, epc_page, secs_page);
225 if (ret) {
226 sgx_encl_free_epc_page(epc_page);
227 return ERR_PTR(ret);
228 }
229
230 sgx_free_va_slot(encl_page->va_page, va_offset);
231 list_move(&encl_page->va_page->list, &encl->va_pages);
232 encl_page->desc &= ~SGX_ENCL_PAGE_VA_OFFSET_MASK;
233 encl_page->epc_page = epc_page;
234
235 return epc_page;
236}
237
238/*
239 * Ensure the SECS page is not swapped out. Must be called with encl->lock
240 * to protect the enclave states including SECS and ensure the SECS page is
241 * not swapped out again while being used.
242 */
243static struct sgx_epc_page *sgx_encl_load_secs(struct sgx_encl *encl)
244{
245 struct sgx_epc_page *epc_page = encl->secs.epc_page;
246
247 if (!epc_page)
248 epc_page = sgx_encl_eldu(&encl->secs, NULL);
249
250 return epc_page;
251}
252
253static struct sgx_encl_page *__sgx_encl_load_page(struct sgx_encl *encl,
254 struct sgx_encl_page *entry)
255{
256 struct sgx_epc_page *epc_page;
257
258 /* Entry successfully located. */
259 if (entry->epc_page) {
260 if (entry->desc & SGX_ENCL_PAGE_BEING_RECLAIMED)
261 return ERR_PTR(-EBUSY);
262
263 return entry;
264 }
265
266 epc_page = sgx_encl_load_secs(encl);
267 if (IS_ERR(epc_page))
268 return ERR_CAST(epc_page);
269
270 epc_page = sgx_encl_eldu(entry, encl->secs.epc_page);
271 if (IS_ERR(epc_page))
272 return ERR_CAST(epc_page);
273
274 encl->secs_child_cnt++;
275 sgx_mark_page_reclaimable(entry->epc_page);
276
277 return entry;
278}
279
280static struct sgx_encl_page *sgx_encl_load_page_in_vma(struct sgx_encl *encl,
281 unsigned long addr,
282 unsigned long vm_flags)
283{
284 unsigned long vm_prot_bits = vm_flags & VM_ACCESS_FLAGS;
285 struct sgx_encl_page *entry;
286
287 entry = xa_load(&encl->page_array, PFN_DOWN(addr));
288 if (!entry)
289 return ERR_PTR(-EFAULT);
290
291 /*
292 * Verify that the page has equal or higher build time
293 * permissions than the VMA permissions (i.e. the subset of {VM_READ,
294 * VM_WRITE, VM_EXECUTE} in vma->vm_flags).
295 */
296 if ((entry->vm_max_prot_bits & vm_prot_bits) != vm_prot_bits)
297 return ERR_PTR(-EFAULT);
298
299 return __sgx_encl_load_page(encl, entry);
300}
301
302struct sgx_encl_page *sgx_encl_load_page(struct sgx_encl *encl,
303 unsigned long addr)
304{
305 struct sgx_encl_page *entry;
306
307 entry = xa_load(&encl->page_array, PFN_DOWN(addr));
308 if (!entry)
309 return ERR_PTR(-EFAULT);
310
311 return __sgx_encl_load_page(encl, entry);
312}
313
314/**
315 * sgx_encl_eaug_page() - Dynamically add page to initialized enclave
316 * @vma: VMA obtained from fault info from where page is accessed
317 * @encl: enclave accessing the page
318 * @addr: address that triggered the page fault
319 *
320 * When an initialized enclave accesses a page with no backing EPC page
321 * on a SGX2 system then the EPC can be added dynamically via the SGX2
322 * ENCLS[EAUG] instruction.
323 *
324 * Returns: Appropriate vm_fault_t: VM_FAULT_NOPAGE when PTE was installed
325 * successfully, VM_FAULT_SIGBUS or VM_FAULT_OOM as error otherwise.
326 */
327static vm_fault_t sgx_encl_eaug_page(struct vm_area_struct *vma,
328 struct sgx_encl *encl, unsigned long addr)
329{
330 vm_fault_t vmret = VM_FAULT_SIGBUS;
331 struct sgx_pageinfo pginfo = {0};
332 struct sgx_encl_page *encl_page;
333 struct sgx_epc_page *epc_page;
334 struct sgx_va_page *va_page;
335 unsigned long phys_addr;
336 u64 secinfo_flags;
337 int ret;
338
339 if (!test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
340 return VM_FAULT_SIGBUS;
341
342 /*
343 * Ignore internal permission checking for dynamically added pages.
344 * They matter only for data added during the pre-initialization
345 * phase. The enclave decides the permissions by the means of
346 * EACCEPT, EACCEPTCOPY and EMODPE.
347 */
348 secinfo_flags = SGX_SECINFO_R | SGX_SECINFO_W | SGX_SECINFO_X;
349 encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags);
350 if (IS_ERR(encl_page))
351 return VM_FAULT_OOM;
352
353 mutex_lock(&encl->lock);
354
355 epc_page = sgx_encl_load_secs(encl);
356 if (IS_ERR(epc_page)) {
357 if (PTR_ERR(epc_page) == -EBUSY)
358 vmret = VM_FAULT_NOPAGE;
359 goto err_out_unlock;
360 }
361
362 epc_page = sgx_alloc_epc_page(encl_page, false);
363 if (IS_ERR(epc_page)) {
364 if (PTR_ERR(epc_page) == -EBUSY)
365 vmret = VM_FAULT_NOPAGE;
366 goto err_out_unlock;
367 }
368
369 va_page = sgx_encl_grow(encl, false);
370 if (IS_ERR(va_page)) {
371 if (PTR_ERR(va_page) == -EBUSY)
372 vmret = VM_FAULT_NOPAGE;
373 goto err_out_epc;
374 }
375
376 if (va_page)
377 list_add(&va_page->list, &encl->va_pages);
378
379 ret = xa_insert(&encl->page_array, PFN_DOWN(encl_page->desc),
380 encl_page, GFP_KERNEL);
381 /*
382 * If ret == -EBUSY then page was created in another flow while
383 * running without encl->lock
384 */
385 if (ret)
386 goto err_out_shrink;
387
388 pginfo.secs = (unsigned long)sgx_get_epc_virt_addr(encl->secs.epc_page);
389 pginfo.addr = encl_page->desc & PAGE_MASK;
390 pginfo.metadata = 0;
391
392 ret = __eaug(&pginfo, sgx_get_epc_virt_addr(epc_page));
393 if (ret)
394 goto err_out;
395
396 encl_page->encl = encl;
397 encl_page->epc_page = epc_page;
398 encl_page->type = SGX_PAGE_TYPE_REG;
399 encl->secs_child_cnt++;
400
401 sgx_mark_page_reclaimable(encl_page->epc_page);
402
403 phys_addr = sgx_get_epc_phys_addr(epc_page);
404 /*
405 * Do not undo everything when creating PTE entry fails - next #PF
406 * would find page ready for a PTE.
407 */
408 vmret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys_addr));
409 if (vmret != VM_FAULT_NOPAGE) {
410 mutex_unlock(&encl->lock);
411 return VM_FAULT_SIGBUS;
412 }
413 mutex_unlock(&encl->lock);
414 return VM_FAULT_NOPAGE;
415
416err_out:
417 xa_erase(&encl->page_array, PFN_DOWN(encl_page->desc));
418
419err_out_shrink:
420 sgx_encl_shrink(encl, va_page);
421err_out_epc:
422 sgx_encl_free_epc_page(epc_page);
423err_out_unlock:
424 mutex_unlock(&encl->lock);
425 kfree(encl_page);
426
427 return vmret;
428}
429
430static vm_fault_t sgx_vma_fault(struct vm_fault *vmf)
431{
432 unsigned long addr = (unsigned long)vmf->address;
433 struct vm_area_struct *vma = vmf->vma;
434 struct sgx_encl_page *entry;
435 unsigned long phys_addr;
436 struct sgx_encl *encl;
437 vm_fault_t ret;
438
439 encl = vma->vm_private_data;
440
441 /*
442 * It's very unlikely but possible that allocating memory for the
443 * mm_list entry of a forked process failed in sgx_vma_open(). When
444 * this happens, vm_private_data is set to NULL.
445 */
446 if (unlikely(!encl))
447 return VM_FAULT_SIGBUS;
448
449 /*
450 * The page_array keeps track of all enclave pages, whether they
451 * are swapped out or not. If there is no entry for this page and
452 * the system supports SGX2 then it is possible to dynamically add
453 * a new enclave page. This is only possible for an initialized
454 * enclave that will be checked for right away.
455 */
456 if (cpu_feature_enabled(X86_FEATURE_SGX2) &&
457 (!xa_load(&encl->page_array, PFN_DOWN(addr))))
458 return sgx_encl_eaug_page(vma, encl, addr);
459
460 mutex_lock(&encl->lock);
461
462 entry = sgx_encl_load_page_in_vma(encl, addr, vma->vm_flags);
463 if (IS_ERR(entry)) {
464 mutex_unlock(&encl->lock);
465
466 if (PTR_ERR(entry) == -EBUSY)
467 return VM_FAULT_NOPAGE;
468
469 return VM_FAULT_SIGBUS;
470 }
471
472 phys_addr = sgx_get_epc_phys_addr(entry->epc_page);
473
474 ret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys_addr));
475 if (ret != VM_FAULT_NOPAGE) {
476 mutex_unlock(&encl->lock);
477
478 return VM_FAULT_SIGBUS;
479 }
480
481 sgx_encl_test_and_clear_young(vma->vm_mm, entry);
482 mutex_unlock(&encl->lock);
483
484 return VM_FAULT_NOPAGE;
485}
486
487static void sgx_vma_open(struct vm_area_struct *vma)
488{
489 struct sgx_encl *encl = vma->vm_private_data;
490
491 /*
492 * It's possible but unlikely that vm_private_data is NULL. This can
493 * happen in a grandchild of a process, when sgx_encl_mm_add() had
494 * failed to allocate memory in this callback.
495 */
496 if (unlikely(!encl))
497 return;
498
499 if (sgx_encl_mm_add(encl, vma->vm_mm))
500 vma->vm_private_data = NULL;
501}
502
503
504/**
505 * sgx_encl_may_map() - Check if a requested VMA mapping is allowed
506 * @encl: an enclave pointer
507 * @start: lower bound of the address range, inclusive
508 * @end: upper bound of the address range, exclusive
509 * @vm_flags: VMA flags
510 *
511 * Iterate through the enclave pages contained within [@start, @end) to verify
512 * that the permissions requested by a subset of {VM_READ, VM_WRITE, VM_EXEC}
513 * do not contain any permissions that are not contained in the build time
514 * permissions of any of the enclave pages within the given address range.
515 *
516 * An enclave creator must declare the strongest permissions that will be
517 * needed for each enclave page. This ensures that mappings have the identical
518 * or weaker permissions than the earlier declared permissions.
519 *
520 * Return: 0 on success, -EACCES otherwise
521 */
522int sgx_encl_may_map(struct sgx_encl *encl, unsigned long start,
523 unsigned long end, unsigned long vm_flags)
524{
525 unsigned long vm_prot_bits = vm_flags & VM_ACCESS_FLAGS;
526 struct sgx_encl_page *page;
527 unsigned long count = 0;
528 int ret = 0;
529
530 XA_STATE(xas, &encl->page_array, PFN_DOWN(start));
531
532 /* Disallow mapping outside enclave's address range. */
533 if (test_bit(SGX_ENCL_INITIALIZED, &encl->flags) &&
534 (start < encl->base || end > encl->base + encl->size))
535 return -EACCES;
536
537 /*
538 * Disallow READ_IMPLIES_EXEC tasks as their VMA permissions might
539 * conflict with the enclave page permissions.
540 */
541 if (current->personality & READ_IMPLIES_EXEC)
542 return -EACCES;
543
544 mutex_lock(&encl->lock);
545 xas_lock(&xas);
546 xas_for_each(&xas, page, PFN_DOWN(end - 1)) {
547 if (~page->vm_max_prot_bits & vm_prot_bits) {
548 ret = -EACCES;
549 break;
550 }
551
552 /* Reschedule on every XA_CHECK_SCHED iteration. */
553 if (!(++count % XA_CHECK_SCHED)) {
554 xas_pause(&xas);
555 xas_unlock(&xas);
556 mutex_unlock(&encl->lock);
557
558 cond_resched();
559
560 mutex_lock(&encl->lock);
561 xas_lock(&xas);
562 }
563 }
564 xas_unlock(&xas);
565 mutex_unlock(&encl->lock);
566
567 return ret;
568}
569
570static int sgx_vma_mprotect(struct vm_area_struct *vma, unsigned long start,
571 unsigned long end, unsigned long newflags)
572{
573 return sgx_encl_may_map(vma->vm_private_data, start, end, newflags);
574}
575
576static int sgx_encl_debug_read(struct sgx_encl *encl, struct sgx_encl_page *page,
577 unsigned long addr, void *data)
578{
579 unsigned long offset = addr & ~PAGE_MASK;
580 int ret;
581
582
583 ret = __edbgrd(sgx_get_epc_virt_addr(page->epc_page) + offset, data);
584 if (ret)
585 return -EIO;
586
587 return 0;
588}
589
590static int sgx_encl_debug_write(struct sgx_encl *encl, struct sgx_encl_page *page,
591 unsigned long addr, void *data)
592{
593 unsigned long offset = addr & ~PAGE_MASK;
594 int ret;
595
596 ret = __edbgwr(sgx_get_epc_virt_addr(page->epc_page) + offset, data);
597 if (ret)
598 return -EIO;
599
600 return 0;
601}
602
603/*
604 * Load an enclave page to EPC if required, and take encl->lock.
605 */
606static struct sgx_encl_page *sgx_encl_reserve_page(struct sgx_encl *encl,
607 unsigned long addr,
608 unsigned long vm_flags)
609{
610 struct sgx_encl_page *entry;
611
612 for ( ; ; ) {
613 mutex_lock(&encl->lock);
614
615 entry = sgx_encl_load_page_in_vma(encl, addr, vm_flags);
616 if (PTR_ERR(entry) != -EBUSY)
617 break;
618
619 mutex_unlock(&encl->lock);
620 }
621
622 if (IS_ERR(entry))
623 mutex_unlock(&encl->lock);
624
625 return entry;
626}
627
628static int sgx_vma_access(struct vm_area_struct *vma, unsigned long addr,
629 void *buf, int len, int write)
630{
631 struct sgx_encl *encl = vma->vm_private_data;
632 struct sgx_encl_page *entry = NULL;
633 char data[sizeof(unsigned long)];
634 unsigned long align;
635 int offset;
636 int cnt;
637 int ret = 0;
638 int i;
639
640 /*
641 * If process was forked, VMA is still there but vm_private_data is set
642 * to NULL.
643 */
644 if (!encl)
645 return -EFAULT;
646
647 if (!test_bit(SGX_ENCL_DEBUG, &encl->flags))
648 return -EFAULT;
649
650 for (i = 0; i < len; i += cnt) {
651 entry = sgx_encl_reserve_page(encl, (addr + i) & PAGE_MASK,
652 vma->vm_flags);
653 if (IS_ERR(entry)) {
654 ret = PTR_ERR(entry);
655 break;
656 }
657
658 align = ALIGN_DOWN(addr + i, sizeof(unsigned long));
659 offset = (addr + i) & (sizeof(unsigned long) - 1);
660 cnt = sizeof(unsigned long) - offset;
661 cnt = min(cnt, len - i);
662
663 ret = sgx_encl_debug_read(encl, entry, align, data);
664 if (ret)
665 goto out;
666
667 if (write) {
668 memcpy(data + offset, buf + i, cnt);
669 ret = sgx_encl_debug_write(encl, entry, align, data);
670 if (ret)
671 goto out;
672 } else {
673 memcpy(buf + i, data + offset, cnt);
674 }
675
676out:
677 mutex_unlock(&encl->lock);
678
679 if (ret)
680 break;
681 }
682
683 return ret < 0 ? ret : i;
684}
685
686const struct vm_operations_struct sgx_vm_ops = {
687 .fault = sgx_vma_fault,
688 .mprotect = sgx_vma_mprotect,
689 .open = sgx_vma_open,
690 .access = sgx_vma_access,
691};
692
693/**
694 * sgx_encl_release - Destroy an enclave instance
695 * @ref: address of a kref inside &sgx_encl
696 *
697 * Used together with kref_put(). Frees all the resources associated with the
698 * enclave and the instance itself.
699 */
700void sgx_encl_release(struct kref *ref)
701{
702 struct sgx_encl *encl = container_of(ref, struct sgx_encl, refcount);
703 unsigned long max_page_index = PFN_DOWN(encl->base + encl->size - 1);
704 struct sgx_va_page *va_page;
705 struct sgx_encl_page *entry;
706 unsigned long count = 0;
707
708 XA_STATE(xas, &encl->page_array, PFN_DOWN(encl->base));
709
710 xas_lock(&xas);
711 xas_for_each(&xas, entry, max_page_index) {
712 if (entry->epc_page) {
713 /*
714 * The page and its radix tree entry cannot be freed
715 * if the page is being held by the reclaimer.
716 */
717 if (sgx_unmark_page_reclaimable(entry->epc_page))
718 continue;
719
720 sgx_encl_free_epc_page(entry->epc_page);
721 encl->secs_child_cnt--;
722 entry->epc_page = NULL;
723 }
724
725 kfree(entry);
726 /*
727 * Invoke scheduler on every XA_CHECK_SCHED iteration
728 * to prevent soft lockups.
729 */
730 if (!(++count % XA_CHECK_SCHED)) {
731 xas_pause(&xas);
732 xas_unlock(&xas);
733
734 cond_resched();
735
736 xas_lock(&xas);
737 }
738 }
739 xas_unlock(&xas);
740
741 xa_destroy(&encl->page_array);
742
743 if (!encl->secs_child_cnt && encl->secs.epc_page) {
744 sgx_encl_free_epc_page(encl->secs.epc_page);
745 encl->secs.epc_page = NULL;
746 }
747
748 while (!list_empty(&encl->va_pages)) {
749 va_page = list_first_entry(&encl->va_pages, struct sgx_va_page,
750 list);
751 list_del(&va_page->list);
752 sgx_encl_free_epc_page(va_page->epc_page);
753 kfree(va_page);
754 }
755
756 if (encl->backing)
757 fput(encl->backing);
758
759 cleanup_srcu_struct(&encl->srcu);
760
761 WARN_ON_ONCE(!list_empty(&encl->mm_list));
762
763 /* Detect EPC page leak's. */
764 WARN_ON_ONCE(encl->secs_child_cnt);
765 WARN_ON_ONCE(encl->secs.epc_page);
766
767 kfree(encl);
768}
769
770/*
771 * 'mm' is exiting and no longer needs mmu notifications.
772 */
773static void sgx_mmu_notifier_release(struct mmu_notifier *mn,
774 struct mm_struct *mm)
775{
776 struct sgx_encl_mm *encl_mm = container_of(mn, struct sgx_encl_mm, mmu_notifier);
777 struct sgx_encl_mm *tmp = NULL;
778 bool found = false;
779
780 /*
781 * The enclave itself can remove encl_mm. Note, objects can't be moved
782 * off an RCU protected list, but deletion is ok.
783 */
784 spin_lock(&encl_mm->encl->mm_lock);
785 list_for_each_entry(tmp, &encl_mm->encl->mm_list, list) {
786 if (tmp == encl_mm) {
787 list_del_rcu(&encl_mm->list);
788 found = true;
789 break;
790 }
791 }
792 spin_unlock(&encl_mm->encl->mm_lock);
793
794 if (found) {
795 synchronize_srcu(&encl_mm->encl->srcu);
796 mmu_notifier_put(mn);
797 }
798}
799
800static void sgx_mmu_notifier_free(struct mmu_notifier *mn)
801{
802 struct sgx_encl_mm *encl_mm = container_of(mn, struct sgx_encl_mm, mmu_notifier);
803
804 /* 'encl_mm' is going away, put encl_mm->encl reference: */
805 kref_put(&encl_mm->encl->refcount, sgx_encl_release);
806
807 kfree(encl_mm);
808}
809
810static const struct mmu_notifier_ops sgx_mmu_notifier_ops = {
811 .release = sgx_mmu_notifier_release,
812 .free_notifier = sgx_mmu_notifier_free,
813};
814
815static struct sgx_encl_mm *sgx_encl_find_mm(struct sgx_encl *encl,
816 struct mm_struct *mm)
817{
818 struct sgx_encl_mm *encl_mm = NULL;
819 struct sgx_encl_mm *tmp;
820 int idx;
821
822 idx = srcu_read_lock(&encl->srcu);
823
824 list_for_each_entry_rcu(tmp, &encl->mm_list, list) {
825 if (tmp->mm == mm) {
826 encl_mm = tmp;
827 break;
828 }
829 }
830
831 srcu_read_unlock(&encl->srcu, idx);
832
833 return encl_mm;
834}
835
836int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm)
837{
838 struct sgx_encl_mm *encl_mm;
839 int ret;
840
841 /*
842 * Even though a single enclave may be mapped into an mm more than once,
843 * each 'mm' only appears once on encl->mm_list. This is guaranteed by
844 * holding the mm's mmap lock for write before an mm can be added or
845 * remove to an encl->mm_list.
846 */
847 mmap_assert_write_locked(mm);
848
849 /*
850 * It's possible that an entry already exists in the mm_list, because it
851 * is removed only on VFS release or process exit.
852 */
853 if (sgx_encl_find_mm(encl, mm))
854 return 0;
855
856 encl_mm = kzalloc(sizeof(*encl_mm), GFP_KERNEL);
857 if (!encl_mm)
858 return -ENOMEM;
859
860 /* Grab a refcount for the encl_mm->encl reference: */
861 kref_get(&encl->refcount);
862 encl_mm->encl = encl;
863 encl_mm->mm = mm;
864 encl_mm->mmu_notifier.ops = &sgx_mmu_notifier_ops;
865
866 ret = __mmu_notifier_register(&encl_mm->mmu_notifier, mm);
867 if (ret) {
868 kfree(encl_mm);
869 return ret;
870 }
871
872 spin_lock(&encl->mm_lock);
873 list_add_rcu(&encl_mm->list, &encl->mm_list);
874 /* Pairs with smp_rmb() in sgx_zap_enclave_ptes(). */
875 smp_wmb();
876 encl->mm_list_version++;
877 spin_unlock(&encl->mm_lock);
878
879 return 0;
880}
881
882/**
883 * sgx_encl_cpumask() - Query which CPUs might be accessing the enclave
884 * @encl: the enclave
885 *
886 * Some SGX functions require that no cached linear-to-physical address
887 * mappings are present before they can succeed. For example, ENCLS[EWB]
888 * copies a page from the enclave page cache to regular main memory but
889 * it fails if it cannot ensure that there are no cached
890 * linear-to-physical address mappings referring to the page.
891 *
892 * SGX hardware flushes all cached linear-to-physical mappings on a CPU
893 * when an enclave is exited via ENCLU[EEXIT] or an Asynchronous Enclave
894 * Exit (AEX). Exiting an enclave will thus ensure cached linear-to-physical
895 * address mappings are cleared but coordination with the tracking done within
896 * the SGX hardware is needed to support the SGX functions that depend on this
897 * cache clearing.
898 *
899 * When the ENCLS[ETRACK] function is issued on an enclave the hardware
900 * tracks threads operating inside the enclave at that time. The SGX
901 * hardware tracking require that all the identified threads must have
902 * exited the enclave in order to flush the mappings before a function such
903 * as ENCLS[EWB] will be permitted
904 *
905 * The following flow is used to support SGX functions that require that
906 * no cached linear-to-physical address mappings are present:
907 * 1) Execute ENCLS[ETRACK] to initiate hardware tracking.
908 * 2) Use this function (sgx_encl_cpumask()) to query which CPUs might be
909 * accessing the enclave.
910 * 3) Send IPI to identified CPUs, kicking them out of the enclave and
911 * thus flushing all locally cached linear-to-physical address mappings.
912 * 4) Execute SGX function.
913 *
914 * Context: It is required to call this function after ENCLS[ETRACK].
915 * This will ensure that if any new mm appears (racing with
916 * sgx_encl_mm_add()) then the new mm will enter into the
917 * enclave with fresh linear-to-physical address mappings.
918 *
919 * It is required that all IPIs are completed before a new
920 * ENCLS[ETRACK] is issued so be sure to protect steps 1 to 3
921 * of the above flow with the enclave's mutex.
922 *
923 * Return: cpumask of CPUs that might be accessing @encl
924 */
925const cpumask_t *sgx_encl_cpumask(struct sgx_encl *encl)
926{
927 cpumask_t *cpumask = &encl->cpumask;
928 struct sgx_encl_mm *encl_mm;
929 int idx;
930
931 cpumask_clear(cpumask);
932
933 idx = srcu_read_lock(&encl->srcu);
934
935 list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) {
936 if (!mmget_not_zero(encl_mm->mm))
937 continue;
938
939 cpumask_or(cpumask, cpumask, mm_cpumask(encl_mm->mm));
940
941 mmput_async(encl_mm->mm);
942 }
943
944 srcu_read_unlock(&encl->srcu, idx);
945
946 return cpumask;
947}
948
949static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl,
950 pgoff_t index)
951{
952 struct address_space *mapping = encl->backing->f_mapping;
953 gfp_t gfpmask = mapping_gfp_mask(mapping);
954
955 return shmem_read_mapping_page_gfp(mapping, index, gfpmask);
956}
957
958/**
959 * __sgx_encl_get_backing() - Pin the backing storage
960 * @encl: an enclave pointer
961 * @page_index: enclave page index
962 * @backing: data for accessing backing storage for the page
963 *
964 * Pin the backing storage pages for storing the encrypted contents and Paging
965 * Crypto MetaData (PCMD) of an enclave page.
966 *
967 * Return:
968 * 0 on success,
969 * -errno otherwise.
970 */
971static int __sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index,
972 struct sgx_backing *backing)
973{
974 pgoff_t page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index);
975 struct page *contents;
976 struct page *pcmd;
977
978 contents = sgx_encl_get_backing_page(encl, page_index);
979 if (IS_ERR(contents))
980 return PTR_ERR(contents);
981
982 pcmd = sgx_encl_get_backing_page(encl, PFN_DOWN(page_pcmd_off));
983 if (IS_ERR(pcmd)) {
984 put_page(contents);
985 return PTR_ERR(pcmd);
986 }
987
988 backing->contents = contents;
989 backing->pcmd = pcmd;
990 backing->pcmd_offset = page_pcmd_off & (PAGE_SIZE - 1);
991
992 return 0;
993}
994
995/*
996 * When called from ksgxd, returns the mem_cgroup of a struct mm stored
997 * in the enclave's mm_list. When not called from ksgxd, just returns
998 * the mem_cgroup of the current task.
999 */
1000static struct mem_cgroup *sgx_encl_get_mem_cgroup(struct sgx_encl *encl)
1001{
1002 struct mem_cgroup *memcg = NULL;
1003 struct sgx_encl_mm *encl_mm;
1004 int idx;
1005
1006 /*
1007 * If called from normal task context, return the mem_cgroup
1008 * of the current task's mm. The remainder of the handling is for
1009 * ksgxd.
1010 */
1011 if (!current_is_ksgxd())
1012 return get_mem_cgroup_from_mm(current->mm);
1013
1014 /*
1015 * Search the enclave's mm_list to find an mm associated with
1016 * this enclave to charge the allocation to.
1017 */
1018 idx = srcu_read_lock(&encl->srcu);
1019
1020 list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) {
1021 if (!mmget_not_zero(encl_mm->mm))
1022 continue;
1023
1024 memcg = get_mem_cgroup_from_mm(encl_mm->mm);
1025
1026 mmput_async(encl_mm->mm);
1027
1028 break;
1029 }
1030
1031 srcu_read_unlock(&encl->srcu, idx);
1032
1033 /*
1034 * In the rare case that there isn't an mm associated with
1035 * the enclave, set memcg to the current active mem_cgroup.
1036 * This will be the root mem_cgroup if there is no active
1037 * mem_cgroup.
1038 */
1039 if (!memcg)
1040 return get_mem_cgroup_from_mm(NULL);
1041
1042 return memcg;
1043}
1044
1045/**
1046 * sgx_encl_alloc_backing() - create a new backing storage page
1047 * @encl: an enclave pointer
1048 * @page_index: enclave page index
1049 * @backing: data for accessing backing storage for the page
1050 *
1051 * When called from ksgxd, sets the active memcg from one of the
1052 * mms in the enclave's mm_list prior to any backing page allocation,
1053 * in order to ensure that shmem page allocations are charged to the
1054 * enclave. Create a backing page for loading data back into an EPC page with
1055 * ELDU. This function takes a reference on a new backing page which
1056 * must be dropped with a corresponding call to sgx_encl_put_backing().
1057 *
1058 * Return:
1059 * 0 on success,
1060 * -errno otherwise.
1061 */
1062int sgx_encl_alloc_backing(struct sgx_encl *encl, unsigned long page_index,
1063 struct sgx_backing *backing)
1064{
1065 struct mem_cgroup *encl_memcg = sgx_encl_get_mem_cgroup(encl);
1066 struct mem_cgroup *memcg = set_active_memcg(encl_memcg);
1067 int ret;
1068
1069 ret = __sgx_encl_get_backing(encl, page_index, backing);
1070
1071 set_active_memcg(memcg);
1072 mem_cgroup_put(encl_memcg);
1073
1074 return ret;
1075}
1076
1077/**
1078 * sgx_encl_lookup_backing() - retrieve an existing backing storage page
1079 * @encl: an enclave pointer
1080 * @page_index: enclave page index
1081 * @backing: data for accessing backing storage for the page
1082 *
1083 * Retrieve a backing page for loading data back into an EPC page with ELDU.
1084 * It is the caller's responsibility to ensure that it is appropriate to use
1085 * sgx_encl_lookup_backing() rather than sgx_encl_alloc_backing(). If lookup is
1086 * not used correctly, this will cause an allocation which is not accounted for.
1087 * This function takes a reference on an existing backing page which must be
1088 * dropped with a corresponding call to sgx_encl_put_backing().
1089 *
1090 * Return:
1091 * 0 on success,
1092 * -errno otherwise.
1093 */
1094static int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index,
1095 struct sgx_backing *backing)
1096{
1097 return __sgx_encl_get_backing(encl, page_index, backing);
1098}
1099
1100/**
1101 * sgx_encl_put_backing() - Unpin the backing storage
1102 * @backing: data for accessing backing storage for the page
1103 */
1104void sgx_encl_put_backing(struct sgx_backing *backing)
1105{
1106 put_page(backing->pcmd);
1107 put_page(backing->contents);
1108}
1109
1110static int sgx_encl_test_and_clear_young_cb(pte_t *ptep, unsigned long addr,
1111 void *data)
1112{
1113 pte_t pte;
1114 int ret;
1115
1116 ret = pte_young(*ptep);
1117 if (ret) {
1118 pte = pte_mkold(*ptep);
1119 set_pte_at((struct mm_struct *)data, addr, ptep, pte);
1120 }
1121
1122 return ret;
1123}
1124
1125/**
1126 * sgx_encl_test_and_clear_young() - Test and reset the accessed bit
1127 * @mm: mm_struct that is checked
1128 * @page: enclave page to be tested for recent access
1129 *
1130 * Checks the Access (A) bit from the PTE corresponding to the enclave page and
1131 * clears it.
1132 *
1133 * Return: 1 if the page has been recently accessed and 0 if not.
1134 */
1135int sgx_encl_test_and_clear_young(struct mm_struct *mm,
1136 struct sgx_encl_page *page)
1137{
1138 unsigned long addr = page->desc & PAGE_MASK;
1139 struct sgx_encl *encl = page->encl;
1140 struct vm_area_struct *vma;
1141 int ret;
1142
1143 ret = sgx_encl_find(mm, addr, &vma);
1144 if (ret)
1145 return 0;
1146
1147 if (encl != vma->vm_private_data)
1148 return 0;
1149
1150 ret = apply_to_page_range(vma->vm_mm, addr, PAGE_SIZE,
1151 sgx_encl_test_and_clear_young_cb, vma->vm_mm);
1152 if (ret < 0)
1153 return 0;
1154
1155 return ret;
1156}
1157
1158struct sgx_encl_page *sgx_encl_page_alloc(struct sgx_encl *encl,
1159 unsigned long offset,
1160 u64 secinfo_flags)
1161{
1162 struct sgx_encl_page *encl_page;
1163 unsigned long prot;
1164
1165 encl_page = kzalloc(sizeof(*encl_page), GFP_KERNEL);
1166 if (!encl_page)
1167 return ERR_PTR(-ENOMEM);
1168
1169 encl_page->desc = encl->base + offset;
1170 encl_page->encl = encl;
1171
1172 prot = _calc_vm_trans(secinfo_flags, SGX_SECINFO_R, PROT_READ) |
1173 _calc_vm_trans(secinfo_flags, SGX_SECINFO_W, PROT_WRITE) |
1174 _calc_vm_trans(secinfo_flags, SGX_SECINFO_X, PROT_EXEC);
1175
1176 /*
1177 * TCS pages must always RW set for CPU access while the SECINFO
1178 * permissions are *always* zero - the CPU ignores the user provided
1179 * values and silently overwrites them with zero permissions.
1180 */
1181 if ((secinfo_flags & SGX_SECINFO_PAGE_TYPE_MASK) == SGX_SECINFO_TCS)
1182 prot |= PROT_READ | PROT_WRITE;
1183
1184 /* Calculate maximum of the VM flags for the page. */
1185 encl_page->vm_max_prot_bits = calc_vm_prot_bits(prot, 0);
1186
1187 return encl_page;
1188}
1189
1190/**
1191 * sgx_zap_enclave_ptes() - remove PTEs mapping the address from enclave
1192 * @encl: the enclave
1193 * @addr: page aligned pointer to single page for which PTEs will be removed
1194 *
1195 * Multiple VMAs may have an enclave page mapped. Remove the PTE mapping
1196 * @addr from each VMA. Ensure that page fault handler is ready to handle
1197 * new mappings of @addr before calling this function.
1198 */
1199void sgx_zap_enclave_ptes(struct sgx_encl *encl, unsigned long addr)
1200{
1201 unsigned long mm_list_version;
1202 struct sgx_encl_mm *encl_mm;
1203 struct vm_area_struct *vma;
1204 int idx, ret;
1205
1206 do {
1207 mm_list_version = encl->mm_list_version;
1208
1209 /* Pairs with smp_wmb() in sgx_encl_mm_add(). */
1210 smp_rmb();
1211
1212 idx = srcu_read_lock(&encl->srcu);
1213
1214 list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) {
1215 if (!mmget_not_zero(encl_mm->mm))
1216 continue;
1217
1218 mmap_read_lock(encl_mm->mm);
1219
1220 ret = sgx_encl_find(encl_mm->mm, addr, &vma);
1221 if (!ret && encl == vma->vm_private_data)
1222 zap_vma_ptes(vma, addr, PAGE_SIZE);
1223
1224 mmap_read_unlock(encl_mm->mm);
1225
1226 mmput_async(encl_mm->mm);
1227 }
1228
1229 srcu_read_unlock(&encl->srcu, idx);
1230 } while (unlikely(encl->mm_list_version != mm_list_version));
1231}
1232
1233/**
1234 * sgx_alloc_va_page() - Allocate a Version Array (VA) page
1235 * @reclaim: Reclaim EPC pages directly if none available. Enclave
1236 * mutex should not be held if this is set.
1237 *
1238 * Allocate a free EPC page and convert it to a Version Array (VA) page.
1239 *
1240 * Return:
1241 * a VA page,
1242 * -errno otherwise
1243 */
1244struct sgx_epc_page *sgx_alloc_va_page(bool reclaim)
1245{
1246 struct sgx_epc_page *epc_page;
1247 int ret;
1248
1249 epc_page = sgx_alloc_epc_page(NULL, reclaim);
1250 if (IS_ERR(epc_page))
1251 return ERR_CAST(epc_page);
1252
1253 ret = __epa(sgx_get_epc_virt_addr(epc_page));
1254 if (ret) {
1255 WARN_ONCE(1, "EPA returned %d (0x%x)", ret, ret);
1256 sgx_encl_free_epc_page(epc_page);
1257 return ERR_PTR(-EFAULT);
1258 }
1259
1260 return epc_page;
1261}
1262
1263/**
1264 * sgx_alloc_va_slot - allocate a VA slot
1265 * @va_page: a &struct sgx_va_page instance
1266 *
1267 * Allocates a slot from a &struct sgx_va_page instance.
1268 *
1269 * Return: offset of the slot inside the VA page
1270 */
1271unsigned int sgx_alloc_va_slot(struct sgx_va_page *va_page)
1272{
1273 int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT);
1274
1275 if (slot < SGX_VA_SLOT_COUNT)
1276 set_bit(slot, va_page->slots);
1277
1278 return slot << 3;
1279}
1280
1281/**
1282 * sgx_free_va_slot - free a VA slot
1283 * @va_page: a &struct sgx_va_page instance
1284 * @offset: offset of the slot inside the VA page
1285 *
1286 * Frees a slot from a &struct sgx_va_page instance.
1287 */
1288void sgx_free_va_slot(struct sgx_va_page *va_page, unsigned int offset)
1289{
1290 clear_bit(offset >> 3, va_page->slots);
1291}
1292
1293/**
1294 * sgx_va_page_full - is the VA page full?
1295 * @va_page: a &struct sgx_va_page instance
1296 *
1297 * Return: true if all slots have been taken
1298 */
1299bool sgx_va_page_full(struct sgx_va_page *va_page)
1300{
1301 int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT);
1302
1303 return slot == SGX_VA_SLOT_COUNT;
1304}
1305
1306/**
1307 * sgx_encl_free_epc_page - free an EPC page assigned to an enclave
1308 * @page: EPC page to be freed
1309 *
1310 * Free an EPC page assigned to an enclave. It does EREMOVE for the page, and
1311 * only upon success, it puts the page back to free page list. Otherwise, it
1312 * gives a WARNING to indicate page is leaked.
1313 */
1314void sgx_encl_free_epc_page(struct sgx_epc_page *page)
1315{
1316 int ret;
1317
1318 WARN_ON_ONCE(page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED);
1319
1320 ret = __eremove(sgx_get_epc_virt_addr(page));
1321 if (WARN_ONCE(ret, EREMOVE_ERROR_MESSAGE, ret, ret))
1322 return;
1323
1324 sgx_free_epc_page(page);
1325}