Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2016-20 Intel Corporation. */
3
4#include <asm/mman.h>
5#include <asm/sgx.h>
6#include <linux/mman.h>
7#include <linux/delay.h>
8#include <linux/file.h>
9#include <linux/hashtable.h>
10#include <linux/highmem.h>
11#include <linux/ratelimit.h>
12#include <linux/sched/signal.h>
13#include <linux/shmem_fs.h>
14#include <linux/slab.h>
15#include <linux/suspend.h>
16#include "driver.h"
17#include "encl.h"
18#include "encls.h"
19
20struct sgx_va_page *sgx_encl_grow(struct sgx_encl *encl, bool reclaim)
21{
22 struct sgx_va_page *va_page = NULL;
23 void *err;
24
25 BUILD_BUG_ON(SGX_VA_SLOT_COUNT !=
26 (SGX_ENCL_PAGE_VA_OFFSET_MASK >> 3) + 1);
27
28 if (!(encl->page_cnt % SGX_VA_SLOT_COUNT)) {
29 va_page = kzalloc(sizeof(*va_page), GFP_KERNEL);
30 if (!va_page)
31 return ERR_PTR(-ENOMEM);
32
33 va_page->epc_page = sgx_alloc_va_page(reclaim);
34 if (IS_ERR(va_page->epc_page)) {
35 err = ERR_CAST(va_page->epc_page);
36 kfree(va_page);
37 return err;
38 }
39
40 WARN_ON_ONCE(encl->page_cnt % SGX_VA_SLOT_COUNT);
41 }
42 encl->page_cnt++;
43 return va_page;
44}
45
46void sgx_encl_shrink(struct sgx_encl *encl, struct sgx_va_page *va_page)
47{
48 encl->page_cnt--;
49
50 if (va_page) {
51 sgx_encl_free_epc_page(va_page->epc_page);
52 list_del(&va_page->list);
53 kfree(va_page);
54 }
55}
56
57static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs)
58{
59 struct sgx_epc_page *secs_epc;
60 struct sgx_va_page *va_page;
61 struct sgx_pageinfo pginfo;
62 struct sgx_secinfo secinfo;
63 unsigned long encl_size;
64 struct file *backing;
65 long ret;
66
67 /*
68 * ECREATE would detect this too, but checking here also ensures
69 * that the 'encl_size' calculations below can never overflow.
70 */
71 if (!is_power_of_2(secs->size))
72 return -EINVAL;
73
74 va_page = sgx_encl_grow(encl, true);
75 if (IS_ERR(va_page))
76 return PTR_ERR(va_page);
77 else if (va_page)
78 list_add(&va_page->list, &encl->va_pages);
79 /* else the tail page of the VA page list had free slots. */
80
81 /* The extra page goes to SECS. */
82 encl_size = secs->size + PAGE_SIZE;
83
84 backing = shmem_file_setup("SGX backing", encl_size + (encl_size >> 5),
85 VM_NORESERVE);
86 if (IS_ERR(backing)) {
87 ret = PTR_ERR(backing);
88 goto err_out_shrink;
89 }
90
91 encl->backing = backing;
92
93 secs_epc = sgx_alloc_epc_page(&encl->secs, true);
94 if (IS_ERR(secs_epc)) {
95 ret = PTR_ERR(secs_epc);
96 goto err_out_backing;
97 }
98
99 encl->secs.epc_page = secs_epc;
100
101 pginfo.addr = 0;
102 pginfo.contents = (unsigned long)secs;
103 pginfo.metadata = (unsigned long)&secinfo;
104 pginfo.secs = 0;
105 memset(&secinfo, 0, sizeof(secinfo));
106
107 ret = __ecreate((void *)&pginfo, sgx_get_epc_virt_addr(secs_epc));
108 if (ret) {
109 ret = -EIO;
110 goto err_out;
111 }
112
113 if (secs->attributes & SGX_ATTR_DEBUG)
114 set_bit(SGX_ENCL_DEBUG, &encl->flags);
115
116 encl->secs.encl = encl;
117 encl->secs.type = SGX_PAGE_TYPE_SECS;
118 encl->base = secs->base;
119 encl->size = secs->size;
120 encl->attributes = secs->attributes;
121 encl->attributes_mask = SGX_ATTR_UNPRIV_MASK;
122
123 /* Set only after completion, as encl->lock has not been taken. */
124 set_bit(SGX_ENCL_CREATED, &encl->flags);
125
126 return 0;
127
128err_out:
129 sgx_encl_free_epc_page(encl->secs.epc_page);
130 encl->secs.epc_page = NULL;
131
132err_out_backing:
133 fput(encl->backing);
134 encl->backing = NULL;
135
136err_out_shrink:
137 sgx_encl_shrink(encl, va_page);
138
139 return ret;
140}
141
142/**
143 * sgx_ioc_enclave_create() - handler for %SGX_IOC_ENCLAVE_CREATE
144 * @encl: An enclave pointer.
145 * @arg: The ioctl argument.
146 *
147 * Allocate kernel data structures for the enclave and invoke ECREATE.
148 *
149 * Return:
150 * - 0: Success.
151 * - -EIO: ECREATE failed.
152 * - -errno: POSIX error.
153 */
154static long sgx_ioc_enclave_create(struct sgx_encl *encl, void __user *arg)
155{
156 struct sgx_enclave_create create_arg;
157 void *secs;
158 int ret;
159
160 if (test_bit(SGX_ENCL_CREATED, &encl->flags))
161 return -EINVAL;
162
163 if (copy_from_user(&create_arg, arg, sizeof(create_arg)))
164 return -EFAULT;
165
166 secs = kmalloc(PAGE_SIZE, GFP_KERNEL);
167 if (!secs)
168 return -ENOMEM;
169
170 if (copy_from_user(secs, (void __user *)create_arg.src, PAGE_SIZE))
171 ret = -EFAULT;
172 else
173 ret = sgx_encl_create(encl, secs);
174
175 kfree(secs);
176 return ret;
177}
178
179static int sgx_validate_secinfo(struct sgx_secinfo *secinfo)
180{
181 u64 perm = secinfo->flags & SGX_SECINFO_PERMISSION_MASK;
182 u64 pt = secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK;
183
184 if (pt != SGX_SECINFO_REG && pt != SGX_SECINFO_TCS)
185 return -EINVAL;
186
187 if ((perm & SGX_SECINFO_W) && !(perm & SGX_SECINFO_R))
188 return -EINVAL;
189
190 /*
191 * CPU will silently overwrite the permissions as zero, which means
192 * that we need to validate it ourselves.
193 */
194 if (pt == SGX_SECINFO_TCS && perm)
195 return -EINVAL;
196
197 if (secinfo->flags & SGX_SECINFO_RESERVED_MASK)
198 return -EINVAL;
199
200 if (memchr_inv(secinfo->reserved, 0, sizeof(secinfo->reserved)))
201 return -EINVAL;
202
203 return 0;
204}
205
206static int __sgx_encl_add_page(struct sgx_encl *encl,
207 struct sgx_encl_page *encl_page,
208 struct sgx_epc_page *epc_page,
209 struct sgx_secinfo *secinfo, unsigned long src)
210{
211 struct sgx_pageinfo pginfo;
212 struct vm_area_struct *vma;
213 struct page *src_page;
214 int ret;
215
216 /* Deny noexec. */
217 vma = find_vma(current->mm, src);
218 if (!vma)
219 return -EFAULT;
220
221 if (!(vma->vm_flags & VM_MAYEXEC))
222 return -EACCES;
223
224 ret = get_user_pages(src, 1, 0, &src_page);
225 if (ret < 1)
226 return -EFAULT;
227
228 pginfo.secs = (unsigned long)sgx_get_epc_virt_addr(encl->secs.epc_page);
229 pginfo.addr = encl_page->desc & PAGE_MASK;
230 pginfo.metadata = (unsigned long)secinfo;
231 pginfo.contents = (unsigned long)kmap_local_page(src_page);
232
233 ret = __eadd(&pginfo, sgx_get_epc_virt_addr(epc_page));
234
235 kunmap_local((void *)pginfo.contents);
236 put_page(src_page);
237
238 return ret ? -EIO : 0;
239}
240
241/*
242 * If the caller requires measurement of the page as a proof for the content,
243 * use EEXTEND to add a measurement for 256 bytes of the page. Repeat this
244 * operation until the entire page is measured."
245 */
246static int __sgx_encl_extend(struct sgx_encl *encl,
247 struct sgx_epc_page *epc_page)
248{
249 unsigned long offset;
250 int ret;
251
252 for (offset = 0; offset < PAGE_SIZE; offset += SGX_EEXTEND_BLOCK_SIZE) {
253 ret = __eextend(sgx_get_epc_virt_addr(encl->secs.epc_page),
254 sgx_get_epc_virt_addr(epc_page) + offset);
255 if (ret) {
256 if (encls_failed(ret))
257 ENCLS_WARN(ret, "EEXTEND");
258
259 return -EIO;
260 }
261 }
262
263 return 0;
264}
265
266static int sgx_encl_add_page(struct sgx_encl *encl, unsigned long src,
267 unsigned long offset, struct sgx_secinfo *secinfo,
268 unsigned long flags)
269{
270 struct sgx_encl_page *encl_page;
271 struct sgx_epc_page *epc_page;
272 struct sgx_va_page *va_page;
273 int ret;
274
275 encl_page = sgx_encl_page_alloc(encl, offset, secinfo->flags);
276 if (IS_ERR(encl_page))
277 return PTR_ERR(encl_page);
278
279 epc_page = sgx_alloc_epc_page(encl_page, true);
280 if (IS_ERR(epc_page)) {
281 kfree(encl_page);
282 return PTR_ERR(epc_page);
283 }
284
285 va_page = sgx_encl_grow(encl, true);
286 if (IS_ERR(va_page)) {
287 ret = PTR_ERR(va_page);
288 goto err_out_free;
289 }
290
291 mmap_read_lock(current->mm);
292 mutex_lock(&encl->lock);
293
294 /*
295 * Adding to encl->va_pages must be done under encl->lock. Ditto for
296 * deleting (via sgx_encl_shrink()) in the error path.
297 */
298 if (va_page)
299 list_add(&va_page->list, &encl->va_pages);
300
301 /*
302 * Insert prior to EADD in case of OOM. EADD modifies MRENCLAVE, i.e.
303 * can't be gracefully unwound, while failure on EADD/EXTEND is limited
304 * to userspace errors (or kernel/hardware bugs).
305 */
306 ret = xa_insert(&encl->page_array, PFN_DOWN(encl_page->desc),
307 encl_page, GFP_KERNEL);
308 if (ret)
309 goto err_out_unlock;
310
311 ret = __sgx_encl_add_page(encl, encl_page, epc_page, secinfo,
312 src);
313 if (ret)
314 goto err_out;
315
316 /*
317 * Complete the "add" before doing the "extend" so that the "add"
318 * isn't in a half-baked state in the extremely unlikely scenario
319 * the enclave will be destroyed in response to EEXTEND failure.
320 */
321 encl_page->encl = encl;
322 encl_page->epc_page = epc_page;
323 encl_page->type = (secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK) >> 8;
324 encl->secs_child_cnt++;
325
326 if (flags & SGX_PAGE_MEASURE) {
327 ret = __sgx_encl_extend(encl, epc_page);
328 if (ret)
329 goto err_out;
330 }
331
332 sgx_mark_page_reclaimable(encl_page->epc_page);
333 mutex_unlock(&encl->lock);
334 mmap_read_unlock(current->mm);
335 return ret;
336
337err_out:
338 xa_erase(&encl->page_array, PFN_DOWN(encl_page->desc));
339
340err_out_unlock:
341 sgx_encl_shrink(encl, va_page);
342 mutex_unlock(&encl->lock);
343 mmap_read_unlock(current->mm);
344
345err_out_free:
346 sgx_encl_free_epc_page(epc_page);
347 kfree(encl_page);
348
349 return ret;
350}
351
352/*
353 * Ensure user provided offset and length values are valid for
354 * an enclave.
355 */
356static int sgx_validate_offset_length(struct sgx_encl *encl,
357 unsigned long offset,
358 unsigned long length)
359{
360 if (!IS_ALIGNED(offset, PAGE_SIZE))
361 return -EINVAL;
362
363 if (!length || !IS_ALIGNED(length, PAGE_SIZE))
364 return -EINVAL;
365
366 if (offset + length < offset)
367 return -EINVAL;
368
369 if (offset + length - PAGE_SIZE >= encl->size)
370 return -EINVAL;
371
372 return 0;
373}
374
375/**
376 * sgx_ioc_enclave_add_pages() - The handler for %SGX_IOC_ENCLAVE_ADD_PAGES
377 * @encl: an enclave pointer
378 * @arg: a user pointer to a struct sgx_enclave_add_pages instance
379 *
380 * Add one or more pages to an uninitialized enclave, and optionally extend the
381 * measurement with the contents of the page. The SECINFO and measurement mask
382 * are applied to all pages.
383 *
384 * A SECINFO for a TCS is required to always contain zero permissions because
385 * CPU silently zeros them. Allowing anything else would cause a mismatch in
386 * the measurement.
387 *
388 * mmap()'s protection bits are capped by the page permissions. For each page
389 * address, the maximum protection bits are computed with the following
390 * heuristics:
391 *
392 * 1. A regular page: PROT_R, PROT_W and PROT_X match the SECINFO permissions.
393 * 2. A TCS page: PROT_R | PROT_W.
394 *
395 * mmap() is not allowed to surpass the minimum of the maximum protection bits
396 * within the given address range.
397 *
398 * The function deinitializes kernel data structures for enclave and returns
399 * -EIO in any of the following conditions:
400 *
401 * - Enclave Page Cache (EPC), the physical memory holding enclaves, has
402 * been invalidated. This will cause EADD and EEXTEND to fail.
403 * - If the source address is corrupted somehow when executing EADD.
404 *
405 * Return:
406 * - 0: Success.
407 * - -EACCES: The source page is located in a noexec partition.
408 * - -ENOMEM: Out of EPC pages.
409 * - -EINTR: The call was interrupted before data was processed.
410 * - -EIO: Either EADD or EEXTEND failed because invalid source address
411 * or power cycle.
412 * - -errno: POSIX error.
413 */
414static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg)
415{
416 struct sgx_enclave_add_pages add_arg;
417 struct sgx_secinfo secinfo;
418 unsigned long c;
419 int ret;
420
421 if (!test_bit(SGX_ENCL_CREATED, &encl->flags) ||
422 test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
423 return -EINVAL;
424
425 if (copy_from_user(&add_arg, arg, sizeof(add_arg)))
426 return -EFAULT;
427
428 if (!IS_ALIGNED(add_arg.src, PAGE_SIZE))
429 return -EINVAL;
430
431 if (sgx_validate_offset_length(encl, add_arg.offset, add_arg.length))
432 return -EINVAL;
433
434 if (copy_from_user(&secinfo, (void __user *)add_arg.secinfo,
435 sizeof(secinfo)))
436 return -EFAULT;
437
438 if (sgx_validate_secinfo(&secinfo))
439 return -EINVAL;
440
441 for (c = 0 ; c < add_arg.length; c += PAGE_SIZE) {
442 if (signal_pending(current)) {
443 if (!c)
444 ret = -ERESTARTSYS;
445
446 break;
447 }
448
449 if (need_resched())
450 cond_resched();
451
452 ret = sgx_encl_add_page(encl, add_arg.src + c, add_arg.offset + c,
453 &secinfo, add_arg.flags);
454 if (ret)
455 break;
456 }
457
458 add_arg.count = c;
459
460 if (copy_to_user(arg, &add_arg, sizeof(add_arg)))
461 return -EFAULT;
462
463 return ret;
464}
465
466static int __sgx_get_key_hash(struct crypto_shash *tfm, const void *modulus,
467 void *hash)
468{
469 SHASH_DESC_ON_STACK(shash, tfm);
470
471 shash->tfm = tfm;
472
473 return crypto_shash_digest(shash, modulus, SGX_MODULUS_SIZE, hash);
474}
475
476static int sgx_get_key_hash(const void *modulus, void *hash)
477{
478 struct crypto_shash *tfm;
479 int ret;
480
481 tfm = crypto_alloc_shash("sha256", 0, CRYPTO_ALG_ASYNC);
482 if (IS_ERR(tfm))
483 return PTR_ERR(tfm);
484
485 ret = __sgx_get_key_hash(tfm, modulus, hash);
486
487 crypto_free_shash(tfm);
488 return ret;
489}
490
491static int sgx_encl_init(struct sgx_encl *encl, struct sgx_sigstruct *sigstruct,
492 void *token)
493{
494 u64 mrsigner[4];
495 int i, j;
496 void *addr;
497 int ret;
498
499 /*
500 * Deny initializing enclaves with attributes (namely provisioning)
501 * that have not been explicitly allowed.
502 */
503 if (encl->attributes & ~encl->attributes_mask)
504 return -EACCES;
505
506 /*
507 * Attributes should not be enforced *only* against what's available on
508 * platform (done in sgx_encl_create) but checked and enforced against
509 * the mask for enforcement in sigstruct. For example an enclave could
510 * opt to sign with AVX bit in xfrm, but still be loadable on a platform
511 * without it if the sigstruct->body.attributes_mask does not turn that
512 * bit on.
513 */
514 if (sigstruct->body.attributes & sigstruct->body.attributes_mask &
515 sgx_attributes_reserved_mask)
516 return -EINVAL;
517
518 if (sigstruct->body.miscselect & sigstruct->body.misc_mask &
519 sgx_misc_reserved_mask)
520 return -EINVAL;
521
522 if (sigstruct->body.xfrm & sigstruct->body.xfrm_mask &
523 sgx_xfrm_reserved_mask)
524 return -EINVAL;
525
526 ret = sgx_get_key_hash(sigstruct->modulus, mrsigner);
527 if (ret)
528 return ret;
529
530 mutex_lock(&encl->lock);
531
532 /*
533 * ENCLS[EINIT] is interruptible because it has such a high latency,
534 * e.g. 50k+ cycles on success. If an IRQ/NMI/SMI becomes pending,
535 * EINIT may fail with SGX_UNMASKED_EVENT so that the event can be
536 * serviced.
537 */
538 for (i = 0; i < SGX_EINIT_SLEEP_COUNT; i++) {
539 for (j = 0; j < SGX_EINIT_SPIN_COUNT; j++) {
540 addr = sgx_get_epc_virt_addr(encl->secs.epc_page);
541
542 preempt_disable();
543
544 sgx_update_lepubkeyhash(mrsigner);
545
546 ret = __einit(sigstruct, token, addr);
547
548 preempt_enable();
549
550 if (ret == SGX_UNMASKED_EVENT)
551 continue;
552 else
553 break;
554 }
555
556 if (ret != SGX_UNMASKED_EVENT)
557 break;
558
559 msleep_interruptible(SGX_EINIT_SLEEP_TIME);
560
561 if (signal_pending(current)) {
562 ret = -ERESTARTSYS;
563 goto err_out;
564 }
565 }
566
567 if (encls_faulted(ret)) {
568 if (encls_failed(ret))
569 ENCLS_WARN(ret, "EINIT");
570
571 ret = -EIO;
572 } else if (ret) {
573 pr_debug("EINIT returned %d\n", ret);
574 ret = -EPERM;
575 } else {
576 set_bit(SGX_ENCL_INITIALIZED, &encl->flags);
577 }
578
579err_out:
580 mutex_unlock(&encl->lock);
581 return ret;
582}
583
584/**
585 * sgx_ioc_enclave_init() - handler for %SGX_IOC_ENCLAVE_INIT
586 * @encl: an enclave pointer
587 * @arg: userspace pointer to a struct sgx_enclave_init instance
588 *
589 * Flush any outstanding enqueued EADD operations and perform EINIT. The
590 * Launch Enclave Public Key Hash MSRs are rewritten as necessary to match
591 * the enclave's MRSIGNER, which is calculated from the provided sigstruct.
592 *
593 * Return:
594 * - 0: Success.
595 * - -EPERM: Invalid SIGSTRUCT.
596 * - -EIO: EINIT failed because of a power cycle.
597 * - -errno: POSIX error.
598 */
599static long sgx_ioc_enclave_init(struct sgx_encl *encl, void __user *arg)
600{
601 struct sgx_sigstruct *sigstruct;
602 struct sgx_enclave_init init_arg;
603 void *token;
604 int ret;
605
606 if (!test_bit(SGX_ENCL_CREATED, &encl->flags) ||
607 test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
608 return -EINVAL;
609
610 if (copy_from_user(&init_arg, arg, sizeof(init_arg)))
611 return -EFAULT;
612
613 /*
614 * 'sigstruct' must be on a page boundary and 'token' on a 512 byte
615 * boundary. kmalloc() will give this alignment when allocating
616 * PAGE_SIZE bytes.
617 */
618 sigstruct = kmalloc(PAGE_SIZE, GFP_KERNEL);
619 if (!sigstruct)
620 return -ENOMEM;
621
622 token = (void *)((unsigned long)sigstruct + PAGE_SIZE / 2);
623 memset(token, 0, SGX_LAUNCH_TOKEN_SIZE);
624
625 if (copy_from_user(sigstruct, (void __user *)init_arg.sigstruct,
626 sizeof(*sigstruct))) {
627 ret = -EFAULT;
628 goto out;
629 }
630
631 /*
632 * A legacy field used with Intel signed enclaves. These used to mean
633 * regular and architectural enclaves. The CPU only accepts these values
634 * but they do not have any other meaning.
635 *
636 * Thus, reject any other values.
637 */
638 if (sigstruct->header.vendor != 0x0000 &&
639 sigstruct->header.vendor != 0x8086) {
640 ret = -EINVAL;
641 goto out;
642 }
643
644 ret = sgx_encl_init(encl, sigstruct, token);
645
646out:
647 kfree(sigstruct);
648 return ret;
649}
650
651/**
652 * sgx_ioc_enclave_provision() - handler for %SGX_IOC_ENCLAVE_PROVISION
653 * @encl: an enclave pointer
654 * @arg: userspace pointer to a struct sgx_enclave_provision instance
655 *
656 * Allow ATTRIBUTE.PROVISION_KEY for an enclave by providing a file handle to
657 * /dev/sgx_provision.
658 *
659 * Return:
660 * - 0: Success.
661 * - -errno: Otherwise.
662 */
663static long sgx_ioc_enclave_provision(struct sgx_encl *encl, void __user *arg)
664{
665 struct sgx_enclave_provision params;
666
667 if (copy_from_user(¶ms, arg, sizeof(params)))
668 return -EFAULT;
669
670 return sgx_set_attribute(&encl->attributes_mask, params.fd);
671}
672
673/*
674 * Ensure enclave is ready for SGX2 functions. Readiness is checked
675 * by ensuring the hardware supports SGX2 and the enclave is initialized
676 * and thus able to handle requests to modify pages within it.
677 */
678static int sgx_ioc_sgx2_ready(struct sgx_encl *encl)
679{
680 if (!(cpu_feature_enabled(X86_FEATURE_SGX2)))
681 return -ENODEV;
682
683 if (!test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
684 return -EINVAL;
685
686 return 0;
687}
688
689/*
690 * Some SGX functions require that no cached linear-to-physical address
691 * mappings are present before they can succeed. Collaborate with
692 * hardware via ENCLS[ETRACK] to ensure that all cached
693 * linear-to-physical address mappings belonging to all threads of
694 * the enclave are cleared. See sgx_encl_cpumask() for details.
695 *
696 * Must be called with enclave's mutex held from the time the
697 * SGX function requiring that no cached linear-to-physical mappings
698 * are present is executed until this ETRACK flow is complete.
699 */
700static int sgx_enclave_etrack(struct sgx_encl *encl)
701{
702 void *epc_virt;
703 int ret;
704
705 epc_virt = sgx_get_epc_virt_addr(encl->secs.epc_page);
706 ret = __etrack(epc_virt);
707 if (ret) {
708 /*
709 * ETRACK only fails when there is an OS issue. For
710 * example, two consecutive ETRACK was sent without
711 * completed IPI between.
712 */
713 pr_err_once("ETRACK returned %d (0x%x)", ret, ret);
714 /*
715 * Send IPIs to kick CPUs out of the enclave and
716 * try ETRACK again.
717 */
718 on_each_cpu_mask(sgx_encl_cpumask(encl), sgx_ipi_cb, NULL, 1);
719 ret = __etrack(epc_virt);
720 if (ret) {
721 pr_err_once("ETRACK repeat returned %d (0x%x)",
722 ret, ret);
723 return -EFAULT;
724 }
725 }
726 on_each_cpu_mask(sgx_encl_cpumask(encl), sgx_ipi_cb, NULL, 1);
727
728 return 0;
729}
730
731/**
732 * sgx_enclave_restrict_permissions() - Restrict EPCM permissions
733 * @encl: Enclave to which the pages belong.
734 * @modp: Checked parameters from user on which pages need modifying and
735 * their new permissions.
736 *
737 * Return:
738 * - 0: Success.
739 * - -errno: Otherwise.
740 */
741static long
742sgx_enclave_restrict_permissions(struct sgx_encl *encl,
743 struct sgx_enclave_restrict_permissions *modp)
744{
745 struct sgx_encl_page *entry;
746 struct sgx_secinfo secinfo;
747 unsigned long addr;
748 unsigned long c;
749 void *epc_virt;
750 int ret;
751
752 memset(&secinfo, 0, sizeof(secinfo));
753 secinfo.flags = modp->permissions & SGX_SECINFO_PERMISSION_MASK;
754
755 for (c = 0 ; c < modp->length; c += PAGE_SIZE) {
756 addr = encl->base + modp->offset + c;
757
758 sgx_reclaim_direct();
759
760 mutex_lock(&encl->lock);
761
762 entry = sgx_encl_load_page(encl, addr);
763 if (IS_ERR(entry)) {
764 ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
765 goto out_unlock;
766 }
767
768 /*
769 * Changing EPCM permissions is only supported on regular
770 * SGX pages. Attempting this change on other pages will
771 * result in #PF.
772 */
773 if (entry->type != SGX_PAGE_TYPE_REG) {
774 ret = -EINVAL;
775 goto out_unlock;
776 }
777
778 /*
779 * Apart from ensuring that read-access remains, do not verify
780 * the permission bits requested. Kernel has no control over
781 * how EPCM permissions can be relaxed from within the enclave.
782 * ENCLS[EMODPR] can only remove existing EPCM permissions,
783 * attempting to set new permissions will be ignored by the
784 * hardware.
785 */
786
787 /* Change EPCM permissions. */
788 epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
789 ret = __emodpr(&secinfo, epc_virt);
790 if (encls_faulted(ret)) {
791 /*
792 * All possible faults should be avoidable:
793 * parameters have been checked, will only change
794 * permissions of a regular page, and no concurrent
795 * SGX1/SGX2 ENCLS instructions since these
796 * are protected with mutex.
797 */
798 pr_err_once("EMODPR encountered exception %d\n",
799 ENCLS_TRAPNR(ret));
800 ret = -EFAULT;
801 goto out_unlock;
802 }
803 if (encls_failed(ret)) {
804 modp->result = ret;
805 ret = -EFAULT;
806 goto out_unlock;
807 }
808
809 ret = sgx_enclave_etrack(encl);
810 if (ret) {
811 ret = -EFAULT;
812 goto out_unlock;
813 }
814
815 mutex_unlock(&encl->lock);
816 }
817
818 ret = 0;
819 goto out;
820
821out_unlock:
822 mutex_unlock(&encl->lock);
823out:
824 modp->count = c;
825
826 return ret;
827}
828
829/**
830 * sgx_ioc_enclave_restrict_permissions() - handler for
831 * %SGX_IOC_ENCLAVE_RESTRICT_PERMISSIONS
832 * @encl: an enclave pointer
833 * @arg: userspace pointer to a &struct sgx_enclave_restrict_permissions
834 * instance
835 *
836 * SGX2 distinguishes between relaxing and restricting the enclave page
837 * permissions maintained by the hardware (EPCM permissions) of pages
838 * belonging to an initialized enclave (after SGX_IOC_ENCLAVE_INIT).
839 *
840 * EPCM permissions cannot be restricted from within the enclave, the enclave
841 * requires the kernel to run the privileged level 0 instructions ENCLS[EMODPR]
842 * and ENCLS[ETRACK]. An attempt to relax EPCM permissions with this call
843 * will be ignored by the hardware.
844 *
845 * Return:
846 * - 0: Success
847 * - -errno: Otherwise
848 */
849static long sgx_ioc_enclave_restrict_permissions(struct sgx_encl *encl,
850 void __user *arg)
851{
852 struct sgx_enclave_restrict_permissions params;
853 long ret;
854
855 ret = sgx_ioc_sgx2_ready(encl);
856 if (ret)
857 return ret;
858
859 if (copy_from_user(¶ms, arg, sizeof(params)))
860 return -EFAULT;
861
862 if (sgx_validate_offset_length(encl, params.offset, params.length))
863 return -EINVAL;
864
865 if (params.permissions & ~SGX_SECINFO_PERMISSION_MASK)
866 return -EINVAL;
867
868 /*
869 * Fail early if invalid permissions requested to prevent ENCLS[EMODPR]
870 * from faulting later when the CPU does the same check.
871 */
872 if ((params.permissions & SGX_SECINFO_W) &&
873 !(params.permissions & SGX_SECINFO_R))
874 return -EINVAL;
875
876 if (params.result || params.count)
877 return -EINVAL;
878
879 ret = sgx_enclave_restrict_permissions(encl, ¶ms);
880
881 if (copy_to_user(arg, ¶ms, sizeof(params)))
882 return -EFAULT;
883
884 return ret;
885}
886
887/**
888 * sgx_enclave_modify_types() - Modify type of SGX enclave pages
889 * @encl: Enclave to which the pages belong.
890 * @modt: Checked parameters from user about which pages need modifying
891 * and their new page type.
892 *
893 * Return:
894 * - 0: Success
895 * - -errno: Otherwise
896 */
897static long sgx_enclave_modify_types(struct sgx_encl *encl,
898 struct sgx_enclave_modify_types *modt)
899{
900 unsigned long max_prot_restore;
901 enum sgx_page_type page_type;
902 struct sgx_encl_page *entry;
903 struct sgx_secinfo secinfo;
904 unsigned long prot;
905 unsigned long addr;
906 unsigned long c;
907 void *epc_virt;
908 int ret;
909
910 page_type = modt->page_type & SGX_PAGE_TYPE_MASK;
911
912 /*
913 * The only new page types allowed by hardware are PT_TCS and PT_TRIM.
914 */
915 if (page_type != SGX_PAGE_TYPE_TCS && page_type != SGX_PAGE_TYPE_TRIM)
916 return -EINVAL;
917
918 memset(&secinfo, 0, sizeof(secinfo));
919
920 secinfo.flags = page_type << 8;
921
922 for (c = 0 ; c < modt->length; c += PAGE_SIZE) {
923 addr = encl->base + modt->offset + c;
924
925 sgx_reclaim_direct();
926
927 mutex_lock(&encl->lock);
928
929 entry = sgx_encl_load_page(encl, addr);
930 if (IS_ERR(entry)) {
931 ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
932 goto out_unlock;
933 }
934
935 /*
936 * Borrow the logic from the Intel SDM. Regular pages
937 * (SGX_PAGE_TYPE_REG) can change type to SGX_PAGE_TYPE_TCS
938 * or SGX_PAGE_TYPE_TRIM but TCS pages can only be trimmed.
939 * CET pages not supported yet.
940 */
941 if (!(entry->type == SGX_PAGE_TYPE_REG ||
942 (entry->type == SGX_PAGE_TYPE_TCS &&
943 page_type == SGX_PAGE_TYPE_TRIM))) {
944 ret = -EINVAL;
945 goto out_unlock;
946 }
947
948 max_prot_restore = entry->vm_max_prot_bits;
949
950 /*
951 * Once a regular page becomes a TCS page it cannot be
952 * changed back. So the maximum allowed protection reflects
953 * the TCS page that is always RW from kernel perspective but
954 * will be inaccessible from within enclave. Before doing
955 * so, do make sure that the new page type continues to
956 * respect the originally vetted page permissions.
957 */
958 if (entry->type == SGX_PAGE_TYPE_REG &&
959 page_type == SGX_PAGE_TYPE_TCS) {
960 if (~entry->vm_max_prot_bits & (VM_READ | VM_WRITE)) {
961 ret = -EPERM;
962 goto out_unlock;
963 }
964 prot = PROT_READ | PROT_WRITE;
965 entry->vm_max_prot_bits = calc_vm_prot_bits(prot, 0);
966
967 /*
968 * Prevent page from being reclaimed while mutex
969 * is released.
970 */
971 if (sgx_unmark_page_reclaimable(entry->epc_page)) {
972 ret = -EAGAIN;
973 goto out_entry_changed;
974 }
975
976 /*
977 * Do not keep encl->lock because of dependency on
978 * mmap_lock acquired in sgx_zap_enclave_ptes().
979 */
980 mutex_unlock(&encl->lock);
981
982 sgx_zap_enclave_ptes(encl, addr);
983
984 mutex_lock(&encl->lock);
985
986 sgx_mark_page_reclaimable(entry->epc_page);
987 }
988
989 /* Change EPC type */
990 epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
991 ret = __emodt(&secinfo, epc_virt);
992 if (encls_faulted(ret)) {
993 /*
994 * All possible faults should be avoidable:
995 * parameters have been checked, will only change
996 * valid page types, and no concurrent
997 * SGX1/SGX2 ENCLS instructions since these are
998 * protected with mutex.
999 */
1000 pr_err_once("EMODT encountered exception %d\n",
1001 ENCLS_TRAPNR(ret));
1002 ret = -EFAULT;
1003 goto out_entry_changed;
1004 }
1005 if (encls_failed(ret)) {
1006 modt->result = ret;
1007 ret = -EFAULT;
1008 goto out_entry_changed;
1009 }
1010
1011 ret = sgx_enclave_etrack(encl);
1012 if (ret) {
1013 ret = -EFAULT;
1014 goto out_unlock;
1015 }
1016
1017 entry->type = page_type;
1018
1019 mutex_unlock(&encl->lock);
1020 }
1021
1022 ret = 0;
1023 goto out;
1024
1025out_entry_changed:
1026 entry->vm_max_prot_bits = max_prot_restore;
1027out_unlock:
1028 mutex_unlock(&encl->lock);
1029out:
1030 modt->count = c;
1031
1032 return ret;
1033}
1034
1035/**
1036 * sgx_ioc_enclave_modify_types() - handler for %SGX_IOC_ENCLAVE_MODIFY_TYPES
1037 * @encl: an enclave pointer
1038 * @arg: userspace pointer to a &struct sgx_enclave_modify_types instance
1039 *
1040 * Ability to change the enclave page type supports the following use cases:
1041 *
1042 * * It is possible to add TCS pages to an enclave by changing the type of
1043 * regular pages (%SGX_PAGE_TYPE_REG) to TCS (%SGX_PAGE_TYPE_TCS) pages.
1044 * With this support the number of threads supported by an initialized
1045 * enclave can be increased dynamically.
1046 *
1047 * * Regular or TCS pages can dynamically be removed from an initialized
1048 * enclave by changing the page type to %SGX_PAGE_TYPE_TRIM. Changing the
1049 * page type to %SGX_PAGE_TYPE_TRIM marks the page for removal with actual
1050 * removal done by handler of %SGX_IOC_ENCLAVE_REMOVE_PAGES ioctl() called
1051 * after ENCLU[EACCEPT] is run on %SGX_PAGE_TYPE_TRIM page from within the
1052 * enclave.
1053 *
1054 * Return:
1055 * - 0: Success
1056 * - -errno: Otherwise
1057 */
1058static long sgx_ioc_enclave_modify_types(struct sgx_encl *encl,
1059 void __user *arg)
1060{
1061 struct sgx_enclave_modify_types params;
1062 long ret;
1063
1064 ret = sgx_ioc_sgx2_ready(encl);
1065 if (ret)
1066 return ret;
1067
1068 if (copy_from_user(¶ms, arg, sizeof(params)))
1069 return -EFAULT;
1070
1071 if (sgx_validate_offset_length(encl, params.offset, params.length))
1072 return -EINVAL;
1073
1074 if (params.page_type & ~SGX_PAGE_TYPE_MASK)
1075 return -EINVAL;
1076
1077 if (params.result || params.count)
1078 return -EINVAL;
1079
1080 ret = sgx_enclave_modify_types(encl, ¶ms);
1081
1082 if (copy_to_user(arg, ¶ms, sizeof(params)))
1083 return -EFAULT;
1084
1085 return ret;
1086}
1087
1088/**
1089 * sgx_encl_remove_pages() - Remove trimmed pages from SGX enclave
1090 * @encl: Enclave to which the pages belong
1091 * @params: Checked parameters from user on which pages need to be removed
1092 *
1093 * Return:
1094 * - 0: Success.
1095 * - -errno: Otherwise.
1096 */
1097static long sgx_encl_remove_pages(struct sgx_encl *encl,
1098 struct sgx_enclave_remove_pages *params)
1099{
1100 struct sgx_encl_page *entry;
1101 struct sgx_secinfo secinfo;
1102 unsigned long addr;
1103 unsigned long c;
1104 void *epc_virt;
1105 int ret;
1106
1107 memset(&secinfo, 0, sizeof(secinfo));
1108 secinfo.flags = SGX_SECINFO_R | SGX_SECINFO_W | SGX_SECINFO_X;
1109
1110 for (c = 0 ; c < params->length; c += PAGE_SIZE) {
1111 addr = encl->base + params->offset + c;
1112
1113 sgx_reclaim_direct();
1114
1115 mutex_lock(&encl->lock);
1116
1117 entry = sgx_encl_load_page(encl, addr);
1118 if (IS_ERR(entry)) {
1119 ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
1120 goto out_unlock;
1121 }
1122
1123 if (entry->type != SGX_PAGE_TYPE_TRIM) {
1124 ret = -EPERM;
1125 goto out_unlock;
1126 }
1127
1128 /*
1129 * ENCLS[EMODPR] is a no-op instruction used to inform if
1130 * ENCLU[EACCEPT] was run from within the enclave. If
1131 * ENCLS[EMODPR] is run with RWX on a trimmed page that is
1132 * not yet accepted then it will return
1133 * %SGX_PAGE_NOT_MODIFIABLE, after the trimmed page is
1134 * accepted the instruction will encounter a page fault.
1135 */
1136 epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
1137 ret = __emodpr(&secinfo, epc_virt);
1138 if (!encls_faulted(ret) || ENCLS_TRAPNR(ret) != X86_TRAP_PF) {
1139 ret = -EPERM;
1140 goto out_unlock;
1141 }
1142
1143 if (sgx_unmark_page_reclaimable(entry->epc_page)) {
1144 ret = -EBUSY;
1145 goto out_unlock;
1146 }
1147
1148 /*
1149 * Do not keep encl->lock because of dependency on
1150 * mmap_lock acquired in sgx_zap_enclave_ptes().
1151 */
1152 mutex_unlock(&encl->lock);
1153
1154 sgx_zap_enclave_ptes(encl, addr);
1155
1156 mutex_lock(&encl->lock);
1157
1158 sgx_encl_free_epc_page(entry->epc_page);
1159 encl->secs_child_cnt--;
1160 entry->epc_page = NULL;
1161 xa_erase(&encl->page_array, PFN_DOWN(entry->desc));
1162 sgx_encl_shrink(encl, NULL);
1163 kfree(entry);
1164
1165 mutex_unlock(&encl->lock);
1166 }
1167
1168 ret = 0;
1169 goto out;
1170
1171out_unlock:
1172 mutex_unlock(&encl->lock);
1173out:
1174 params->count = c;
1175
1176 return ret;
1177}
1178
1179/**
1180 * sgx_ioc_enclave_remove_pages() - handler for %SGX_IOC_ENCLAVE_REMOVE_PAGES
1181 * @encl: an enclave pointer
1182 * @arg: userspace pointer to &struct sgx_enclave_remove_pages instance
1183 *
1184 * Final step of the flow removing pages from an initialized enclave. The
1185 * complete flow is:
1186 *
1187 * 1) User changes the type of the pages to be removed to %SGX_PAGE_TYPE_TRIM
1188 * using the %SGX_IOC_ENCLAVE_MODIFY_TYPES ioctl().
1189 * 2) User approves the page removal by running ENCLU[EACCEPT] from within
1190 * the enclave.
1191 * 3) User initiates actual page removal using the
1192 * %SGX_IOC_ENCLAVE_REMOVE_PAGES ioctl() that is handled here.
1193 *
1194 * First remove any page table entries pointing to the page and then proceed
1195 * with the actual removal of the enclave page and data in support of it.
1196 *
1197 * VA pages are not affected by this removal. It is thus possible that the
1198 * enclave may end up with more VA pages than needed to support all its
1199 * pages.
1200 *
1201 * Return:
1202 * - 0: Success
1203 * - -errno: Otherwise
1204 */
1205static long sgx_ioc_enclave_remove_pages(struct sgx_encl *encl,
1206 void __user *arg)
1207{
1208 struct sgx_enclave_remove_pages params;
1209 long ret;
1210
1211 ret = sgx_ioc_sgx2_ready(encl);
1212 if (ret)
1213 return ret;
1214
1215 if (copy_from_user(¶ms, arg, sizeof(params)))
1216 return -EFAULT;
1217
1218 if (sgx_validate_offset_length(encl, params.offset, params.length))
1219 return -EINVAL;
1220
1221 if (params.count)
1222 return -EINVAL;
1223
1224 ret = sgx_encl_remove_pages(encl, ¶ms);
1225
1226 if (copy_to_user(arg, ¶ms, sizeof(params)))
1227 return -EFAULT;
1228
1229 return ret;
1230}
1231
1232long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
1233{
1234 struct sgx_encl *encl = filep->private_data;
1235 int ret;
1236
1237 if (test_and_set_bit(SGX_ENCL_IOCTL, &encl->flags))
1238 return -EBUSY;
1239
1240 switch (cmd) {
1241 case SGX_IOC_ENCLAVE_CREATE:
1242 ret = sgx_ioc_enclave_create(encl, (void __user *)arg);
1243 break;
1244 case SGX_IOC_ENCLAVE_ADD_PAGES:
1245 ret = sgx_ioc_enclave_add_pages(encl, (void __user *)arg);
1246 break;
1247 case SGX_IOC_ENCLAVE_INIT:
1248 ret = sgx_ioc_enclave_init(encl, (void __user *)arg);
1249 break;
1250 case SGX_IOC_ENCLAVE_PROVISION:
1251 ret = sgx_ioc_enclave_provision(encl, (void __user *)arg);
1252 break;
1253 case SGX_IOC_ENCLAVE_RESTRICT_PERMISSIONS:
1254 ret = sgx_ioc_enclave_restrict_permissions(encl,
1255 (void __user *)arg);
1256 break;
1257 case SGX_IOC_ENCLAVE_MODIFY_TYPES:
1258 ret = sgx_ioc_enclave_modify_types(encl, (void __user *)arg);
1259 break;
1260 case SGX_IOC_ENCLAVE_REMOVE_PAGES:
1261 ret = sgx_ioc_enclave_remove_pages(encl, (void __user *)arg);
1262 break;
1263 default:
1264 ret = -ENOIOCTLCMD;
1265 break;
1266 }
1267
1268 clear_bit(SGX_ENCL_IOCTL, &encl->flags);
1269 return ret;
1270}