Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Common Ultravisor functions and initialization
4 *
5 * Copyright IBM Corp. 2019, 2020
6 */
7#define KMSG_COMPONENT "prot_virt"
8#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
9
10#include <linux/kernel.h>
11#include <linux/types.h>
12#include <linux/sizes.h>
13#include <linux/bitmap.h>
14#include <linux/memblock.h>
15#include <linux/pagemap.h>
16#include <linux/swap.h>
17#include <asm/facility.h>
18#include <asm/sections.h>
19#include <asm/uv.h>
20
21/* the bootdata_preserved fields come from ones in arch/s390/boot/uv.c */
22#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
23int __bootdata_preserved(prot_virt_guest);
24#endif
25
26struct uv_info __bootdata_preserved(uv_info);
27
28#if IS_ENABLED(CONFIG_KVM)
29int __bootdata_preserved(prot_virt_host);
30EXPORT_SYMBOL(prot_virt_host);
31EXPORT_SYMBOL(uv_info);
32
33static int __init uv_init(phys_addr_t stor_base, unsigned long stor_len)
34{
35 struct uv_cb_init uvcb = {
36 .header.cmd = UVC_CMD_INIT_UV,
37 .header.len = sizeof(uvcb),
38 .stor_origin = stor_base,
39 .stor_len = stor_len,
40 };
41
42 if (uv_call(0, (uint64_t)&uvcb)) {
43 pr_err("Ultravisor init failed with rc: 0x%x rrc: 0%x\n",
44 uvcb.header.rc, uvcb.header.rrc);
45 return -1;
46 }
47 return 0;
48}
49
50void __init setup_uv(void)
51{
52 void *uv_stor_base;
53
54 if (!is_prot_virt_host())
55 return;
56
57 uv_stor_base = memblock_alloc_try_nid(
58 uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
59 MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
60 if (!uv_stor_base) {
61 pr_warn("Failed to reserve %lu bytes for ultravisor base storage\n",
62 uv_info.uv_base_stor_len);
63 goto fail;
64 }
65
66 if (uv_init(__pa(uv_stor_base), uv_info.uv_base_stor_len)) {
67 memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
68 goto fail;
69 }
70
71 pr_info("Reserving %luMB as ultravisor base storage\n",
72 uv_info.uv_base_stor_len >> 20);
73 return;
74fail:
75 pr_info("Disabling support for protected virtualization");
76 prot_virt_host = 0;
77}
78
79/*
80 * Requests the Ultravisor to pin the page in the shared state. This will
81 * cause an intercept when the guest attempts to unshare the pinned page.
82 */
83static int uv_pin_shared(unsigned long paddr)
84{
85 struct uv_cb_cfs uvcb = {
86 .header.cmd = UVC_CMD_PIN_PAGE_SHARED,
87 .header.len = sizeof(uvcb),
88 .paddr = paddr,
89 };
90
91 if (uv_call(0, (u64)&uvcb))
92 return -EINVAL;
93 return 0;
94}
95
96/*
97 * Requests the Ultravisor to destroy a guest page and make it
98 * accessible to the host. The destroy clears the page instead of
99 * exporting.
100 *
101 * @paddr: Absolute host address of page to be destroyed
102 */
103static int uv_destroy_page(unsigned long paddr)
104{
105 struct uv_cb_cfs uvcb = {
106 .header.cmd = UVC_CMD_DESTR_SEC_STOR,
107 .header.len = sizeof(uvcb),
108 .paddr = paddr
109 };
110
111 if (uv_call(0, (u64)&uvcb)) {
112 /*
113 * Older firmware uses 107/d as an indication of a non secure
114 * page. Let us emulate the newer variant (no-op).
115 */
116 if (uvcb.header.rc == 0x107 && uvcb.header.rrc == 0xd)
117 return 0;
118 return -EINVAL;
119 }
120 return 0;
121}
122
123/*
124 * The caller must already hold a reference to the page
125 */
126int uv_destroy_owned_page(unsigned long paddr)
127{
128 struct page *page = phys_to_page(paddr);
129 int rc;
130
131 get_page(page);
132 rc = uv_destroy_page(paddr);
133 if (!rc)
134 clear_bit(PG_arch_1, &page->flags);
135 put_page(page);
136 return rc;
137}
138
139/*
140 * Requests the Ultravisor to encrypt a guest page and make it
141 * accessible to the host for paging (export).
142 *
143 * @paddr: Absolute host address of page to be exported
144 */
145int uv_convert_from_secure(unsigned long paddr)
146{
147 struct uv_cb_cfs uvcb = {
148 .header.cmd = UVC_CMD_CONV_FROM_SEC_STOR,
149 .header.len = sizeof(uvcb),
150 .paddr = paddr
151 };
152
153 if (uv_call(0, (u64)&uvcb))
154 return -EINVAL;
155 return 0;
156}
157
158/*
159 * The caller must already hold a reference to the page
160 */
161int uv_convert_owned_from_secure(unsigned long paddr)
162{
163 struct page *page = phys_to_page(paddr);
164 int rc;
165
166 get_page(page);
167 rc = uv_convert_from_secure(paddr);
168 if (!rc)
169 clear_bit(PG_arch_1, &page->flags);
170 put_page(page);
171 return rc;
172}
173
174/*
175 * Calculate the expected ref_count for a page that would otherwise have no
176 * further pins. This was cribbed from similar functions in other places in
177 * the kernel, but with some slight modifications. We know that a secure
178 * page can not be a huge page for example.
179 */
180static int expected_page_refs(struct page *page)
181{
182 int res;
183
184 res = page_mapcount(page);
185 if (PageSwapCache(page)) {
186 res++;
187 } else if (page_mapping(page)) {
188 res++;
189 if (page_has_private(page))
190 res++;
191 }
192 return res;
193}
194
195static int make_secure_pte(pte_t *ptep, unsigned long addr,
196 struct page *exp_page, struct uv_cb_header *uvcb)
197{
198 pte_t entry = READ_ONCE(*ptep);
199 struct page *page;
200 int expected, cc = 0;
201
202 if (!pte_present(entry))
203 return -ENXIO;
204 if (pte_val(entry) & _PAGE_INVALID)
205 return -ENXIO;
206
207 page = pte_page(entry);
208 if (page != exp_page)
209 return -ENXIO;
210 if (PageWriteback(page))
211 return -EAGAIN;
212 expected = expected_page_refs(page);
213 if (!page_ref_freeze(page, expected))
214 return -EBUSY;
215 set_bit(PG_arch_1, &page->flags);
216 /*
217 * If the UVC does not succeed or fail immediately, we don't want to
218 * loop for long, or we might get stall notifications.
219 * On the other hand, this is a complex scenario and we are holding a lot of
220 * locks, so we can't easily sleep and reschedule. We try only once,
221 * and if the UVC returned busy or partial completion, we return
222 * -EAGAIN and we let the callers deal with it.
223 */
224 cc = __uv_call(0, (u64)uvcb);
225 page_ref_unfreeze(page, expected);
226 /*
227 * Return -ENXIO if the page was not mapped, -EINVAL for other errors.
228 * If busy or partially completed, return -EAGAIN.
229 */
230 if (cc == UVC_CC_OK)
231 return 0;
232 else if (cc == UVC_CC_BUSY || cc == UVC_CC_PARTIAL)
233 return -EAGAIN;
234 return uvcb->rc == 0x10a ? -ENXIO : -EINVAL;
235}
236
237/**
238 * should_export_before_import - Determine whether an export is needed
239 * before an import-like operation
240 * @uvcb: the Ultravisor control block of the UVC to be performed
241 * @mm: the mm of the process
242 *
243 * Returns whether an export is needed before every import-like operation.
244 * This is needed for shared pages, which don't trigger a secure storage
245 * exception when accessed from a different guest.
246 *
247 * Although considered as one, the Unpin Page UVC is not an actual import,
248 * so it is not affected.
249 *
250 * No export is needed also when there is only one protected VM, because the
251 * page cannot belong to the wrong VM in that case (there is no "other VM"
252 * it can belong to).
253 *
254 * Return: true if an export is needed before every import, otherwise false.
255 */
256static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_struct *mm)
257{
258 /*
259 * The misc feature indicates, among other things, that importing a
260 * shared page from a different protected VM will automatically also
261 * transfer its ownership.
262 */
263 if (test_bit_inv(BIT_UV_FEAT_MISC, &uv_info.uv_feature_indications))
264 return false;
265 if (uvcb->cmd == UVC_CMD_UNPIN_PAGE_SHARED)
266 return false;
267 return atomic_read(&mm->context.protected_count) > 1;
268}
269
270/*
271 * Requests the Ultravisor to make a page accessible to a guest.
272 * If it's brought in the first time, it will be cleared. If
273 * it has been exported before, it will be decrypted and integrity
274 * checked.
275 */
276int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb)
277{
278 struct vm_area_struct *vma;
279 bool local_drain = false;
280 spinlock_t *ptelock;
281 unsigned long uaddr;
282 struct page *page;
283 pte_t *ptep;
284 int rc;
285
286again:
287 rc = -EFAULT;
288 mmap_read_lock(gmap->mm);
289
290 uaddr = __gmap_translate(gmap, gaddr);
291 if (IS_ERR_VALUE(uaddr))
292 goto out;
293 vma = vma_lookup(gmap->mm, uaddr);
294 if (!vma)
295 goto out;
296 /*
297 * Secure pages cannot be huge and userspace should not combine both.
298 * In case userspace does it anyway this will result in an -EFAULT for
299 * the unpack. The guest is thus never reaching secure mode. If
300 * userspace is playing dirty tricky with mapping huge pages later
301 * on this will result in a segmentation fault.
302 */
303 if (is_vm_hugetlb_page(vma))
304 goto out;
305
306 rc = -ENXIO;
307 page = follow_page(vma, uaddr, FOLL_WRITE);
308 if (IS_ERR_OR_NULL(page))
309 goto out;
310
311 lock_page(page);
312 ptep = get_locked_pte(gmap->mm, uaddr, &ptelock);
313 if (should_export_before_import(uvcb, gmap->mm))
314 uv_convert_from_secure(page_to_phys(page));
315 rc = make_secure_pte(ptep, uaddr, page, uvcb);
316 pte_unmap_unlock(ptep, ptelock);
317 unlock_page(page);
318out:
319 mmap_read_unlock(gmap->mm);
320
321 if (rc == -EAGAIN) {
322 /*
323 * If we are here because the UVC returned busy or partial
324 * completion, this is just a useless check, but it is safe.
325 */
326 wait_on_page_writeback(page);
327 } else if (rc == -EBUSY) {
328 /*
329 * If we have tried a local drain and the page refcount
330 * still does not match our expected safe value, try with a
331 * system wide drain. This is needed if the pagevecs holding
332 * the page are on a different CPU.
333 */
334 if (local_drain) {
335 lru_add_drain_all();
336 /* We give up here, and let the caller try again */
337 return -EAGAIN;
338 }
339 /*
340 * We are here if the page refcount does not match the
341 * expected safe value. The main culprits are usually
342 * pagevecs. With lru_add_drain() we drain the pagevecs
343 * on the local CPU so that hopefully the refcount will
344 * reach the expected safe value.
345 */
346 lru_add_drain();
347 local_drain = true;
348 /* And now we try again immediately after draining */
349 goto again;
350 } else if (rc == -ENXIO) {
351 if (gmap_fault(gmap, gaddr, FAULT_FLAG_WRITE))
352 return -EFAULT;
353 return -EAGAIN;
354 }
355 return rc;
356}
357EXPORT_SYMBOL_GPL(gmap_make_secure);
358
359int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr)
360{
361 struct uv_cb_cts uvcb = {
362 .header.cmd = UVC_CMD_CONV_TO_SEC_STOR,
363 .header.len = sizeof(uvcb),
364 .guest_handle = gmap->guest_handle,
365 .gaddr = gaddr,
366 };
367
368 return gmap_make_secure(gmap, gaddr, &uvcb);
369}
370EXPORT_SYMBOL_GPL(gmap_convert_to_secure);
371
372/**
373 * gmap_destroy_page - Destroy a guest page.
374 * @gmap: the gmap of the guest
375 * @gaddr: the guest address to destroy
376 *
377 * An attempt will be made to destroy the given guest page. If the attempt
378 * fails, an attempt is made to export the page. If both attempts fail, an
379 * appropriate error is returned.
380 */
381int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr)
382{
383 struct vm_area_struct *vma;
384 unsigned long uaddr;
385 struct page *page;
386 int rc;
387
388 rc = -EFAULT;
389 mmap_read_lock(gmap->mm);
390
391 uaddr = __gmap_translate(gmap, gaddr);
392 if (IS_ERR_VALUE(uaddr))
393 goto out;
394 vma = vma_lookup(gmap->mm, uaddr);
395 if (!vma)
396 goto out;
397 /*
398 * Huge pages should not be able to become secure
399 */
400 if (is_vm_hugetlb_page(vma))
401 goto out;
402
403 rc = 0;
404 /* we take an extra reference here */
405 page = follow_page(vma, uaddr, FOLL_WRITE | FOLL_GET);
406 if (IS_ERR_OR_NULL(page))
407 goto out;
408 rc = uv_destroy_owned_page(page_to_phys(page));
409 /*
410 * Fault handlers can race; it is possible that two CPUs will fault
411 * on the same secure page. One CPU can destroy the page, reboot,
412 * re-enter secure mode and import it, while the second CPU was
413 * stuck at the beginning of the handler. At some point the second
414 * CPU will be able to progress, and it will not be able to destroy
415 * the page. In that case we do not want to terminate the process,
416 * we instead try to export the page.
417 */
418 if (rc)
419 rc = uv_convert_owned_from_secure(page_to_phys(page));
420 put_page(page);
421out:
422 mmap_read_unlock(gmap->mm);
423 return rc;
424}
425EXPORT_SYMBOL_GPL(gmap_destroy_page);
426
427/*
428 * To be called with the page locked or with an extra reference! This will
429 * prevent gmap_make_secure from touching the page concurrently. Having 2
430 * parallel make_page_accessible is fine, as the UV calls will become a
431 * no-op if the page is already exported.
432 */
433int arch_make_page_accessible(struct page *page)
434{
435 int rc = 0;
436
437 /* Hugepage cannot be protected, so nothing to do */
438 if (PageHuge(page))
439 return 0;
440
441 /*
442 * PG_arch_1 is used in 3 places:
443 * 1. for kernel page tables during early boot
444 * 2. for storage keys of huge pages and KVM
445 * 3. As an indication that this page might be secure. This can
446 * overindicate, e.g. we set the bit before calling
447 * convert_to_secure.
448 * As secure pages are never huge, all 3 variants can co-exists.
449 */
450 if (!test_bit(PG_arch_1, &page->flags))
451 return 0;
452
453 rc = uv_pin_shared(page_to_phys(page));
454 if (!rc) {
455 clear_bit(PG_arch_1, &page->flags);
456 return 0;
457 }
458
459 rc = uv_convert_from_secure(page_to_phys(page));
460 if (!rc) {
461 clear_bit(PG_arch_1, &page->flags);
462 return 0;
463 }
464
465 return rc;
466}
467EXPORT_SYMBOL_GPL(arch_make_page_accessible);
468
469#endif
470
471#if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM)
472static ssize_t uv_query_facilities(struct kobject *kobj,
473 struct kobj_attribute *attr, char *page)
474{
475 return scnprintf(page, PAGE_SIZE, "%lx\n%lx\n%lx\n%lx\n",
476 uv_info.inst_calls_list[0],
477 uv_info.inst_calls_list[1],
478 uv_info.inst_calls_list[2],
479 uv_info.inst_calls_list[3]);
480}
481
482static struct kobj_attribute uv_query_facilities_attr =
483 __ATTR(facilities, 0444, uv_query_facilities, NULL);
484
485static ssize_t uv_query_supp_se_hdr_ver(struct kobject *kobj,
486 struct kobj_attribute *attr, char *buf)
487{
488 return sysfs_emit(buf, "%lx\n", uv_info.supp_se_hdr_ver);
489}
490
491static struct kobj_attribute uv_query_supp_se_hdr_ver_attr =
492 __ATTR(supp_se_hdr_ver, 0444, uv_query_supp_se_hdr_ver, NULL);
493
494static ssize_t uv_query_supp_se_hdr_pcf(struct kobject *kobj,
495 struct kobj_attribute *attr, char *buf)
496{
497 return sysfs_emit(buf, "%lx\n", uv_info.supp_se_hdr_pcf);
498}
499
500static struct kobj_attribute uv_query_supp_se_hdr_pcf_attr =
501 __ATTR(supp_se_hdr_pcf, 0444, uv_query_supp_se_hdr_pcf, NULL);
502
503static ssize_t uv_query_dump_cpu_len(struct kobject *kobj,
504 struct kobj_attribute *attr, char *page)
505{
506 return scnprintf(page, PAGE_SIZE, "%lx\n",
507 uv_info.guest_cpu_stor_len);
508}
509
510static struct kobj_attribute uv_query_dump_cpu_len_attr =
511 __ATTR(uv_query_dump_cpu_len, 0444, uv_query_dump_cpu_len, NULL);
512
513static ssize_t uv_query_dump_storage_state_len(struct kobject *kobj,
514 struct kobj_attribute *attr, char *page)
515{
516 return scnprintf(page, PAGE_SIZE, "%lx\n",
517 uv_info.conf_dump_storage_state_len);
518}
519
520static struct kobj_attribute uv_query_dump_storage_state_len_attr =
521 __ATTR(dump_storage_state_len, 0444, uv_query_dump_storage_state_len, NULL);
522
523static ssize_t uv_query_dump_finalize_len(struct kobject *kobj,
524 struct kobj_attribute *attr, char *page)
525{
526 return scnprintf(page, PAGE_SIZE, "%lx\n",
527 uv_info.conf_dump_finalize_len);
528}
529
530static struct kobj_attribute uv_query_dump_finalize_len_attr =
531 __ATTR(dump_finalize_len, 0444, uv_query_dump_finalize_len, NULL);
532
533static ssize_t uv_query_feature_indications(struct kobject *kobj,
534 struct kobj_attribute *attr, char *buf)
535{
536 return sysfs_emit(buf, "%lx\n", uv_info.uv_feature_indications);
537}
538
539static struct kobj_attribute uv_query_feature_indications_attr =
540 __ATTR(feature_indications, 0444, uv_query_feature_indications, NULL);
541
542static ssize_t uv_query_max_guest_cpus(struct kobject *kobj,
543 struct kobj_attribute *attr, char *page)
544{
545 return scnprintf(page, PAGE_SIZE, "%d\n",
546 uv_info.max_guest_cpu_id + 1);
547}
548
549static struct kobj_attribute uv_query_max_guest_cpus_attr =
550 __ATTR(max_cpus, 0444, uv_query_max_guest_cpus, NULL);
551
552static ssize_t uv_query_max_guest_vms(struct kobject *kobj,
553 struct kobj_attribute *attr, char *page)
554{
555 return scnprintf(page, PAGE_SIZE, "%d\n",
556 uv_info.max_num_sec_conf);
557}
558
559static struct kobj_attribute uv_query_max_guest_vms_attr =
560 __ATTR(max_guests, 0444, uv_query_max_guest_vms, NULL);
561
562static ssize_t uv_query_max_guest_addr(struct kobject *kobj,
563 struct kobj_attribute *attr, char *page)
564{
565 return scnprintf(page, PAGE_SIZE, "%lx\n",
566 uv_info.max_sec_stor_addr);
567}
568
569static struct kobj_attribute uv_query_max_guest_addr_attr =
570 __ATTR(max_address, 0444, uv_query_max_guest_addr, NULL);
571
572static ssize_t uv_query_supp_att_req_hdr_ver(struct kobject *kobj,
573 struct kobj_attribute *attr, char *page)
574{
575 return scnprintf(page, PAGE_SIZE, "%lx\n", uv_info.supp_att_req_hdr_ver);
576}
577
578static struct kobj_attribute uv_query_supp_att_req_hdr_ver_attr =
579 __ATTR(supp_att_req_hdr_ver, 0444, uv_query_supp_att_req_hdr_ver, NULL);
580
581static ssize_t uv_query_supp_att_pflags(struct kobject *kobj,
582 struct kobj_attribute *attr, char *page)
583{
584 return scnprintf(page, PAGE_SIZE, "%lx\n", uv_info.supp_att_pflags);
585}
586
587static struct kobj_attribute uv_query_supp_att_pflags_attr =
588 __ATTR(supp_att_pflags, 0444, uv_query_supp_att_pflags, NULL);
589
590static struct attribute *uv_query_attrs[] = {
591 &uv_query_facilities_attr.attr,
592 &uv_query_feature_indications_attr.attr,
593 &uv_query_max_guest_cpus_attr.attr,
594 &uv_query_max_guest_vms_attr.attr,
595 &uv_query_max_guest_addr_attr.attr,
596 &uv_query_supp_se_hdr_ver_attr.attr,
597 &uv_query_supp_se_hdr_pcf_attr.attr,
598 &uv_query_dump_storage_state_len_attr.attr,
599 &uv_query_dump_finalize_len_attr.attr,
600 &uv_query_dump_cpu_len_attr.attr,
601 &uv_query_supp_att_req_hdr_ver_attr.attr,
602 &uv_query_supp_att_pflags_attr.attr,
603 NULL,
604};
605
606static struct attribute_group uv_query_attr_group = {
607 .attrs = uv_query_attrs,
608};
609
610static ssize_t uv_is_prot_virt_guest(struct kobject *kobj,
611 struct kobj_attribute *attr, char *page)
612{
613 int val = 0;
614
615#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
616 val = prot_virt_guest;
617#endif
618 return scnprintf(page, PAGE_SIZE, "%d\n", val);
619}
620
621static ssize_t uv_is_prot_virt_host(struct kobject *kobj,
622 struct kobj_attribute *attr, char *page)
623{
624 int val = 0;
625
626#if IS_ENABLED(CONFIG_KVM)
627 val = prot_virt_host;
628#endif
629
630 return scnprintf(page, PAGE_SIZE, "%d\n", val);
631}
632
633static struct kobj_attribute uv_prot_virt_guest =
634 __ATTR(prot_virt_guest, 0444, uv_is_prot_virt_guest, NULL);
635
636static struct kobj_attribute uv_prot_virt_host =
637 __ATTR(prot_virt_host, 0444, uv_is_prot_virt_host, NULL);
638
639static const struct attribute *uv_prot_virt_attrs[] = {
640 &uv_prot_virt_guest.attr,
641 &uv_prot_virt_host.attr,
642 NULL,
643};
644
645static struct kset *uv_query_kset;
646static struct kobject *uv_kobj;
647
648static int __init uv_info_init(void)
649{
650 int rc = -ENOMEM;
651
652 if (!test_facility(158))
653 return 0;
654
655 uv_kobj = kobject_create_and_add("uv", firmware_kobj);
656 if (!uv_kobj)
657 return -ENOMEM;
658
659 rc = sysfs_create_files(uv_kobj, uv_prot_virt_attrs);
660 if (rc)
661 goto out_kobj;
662
663 uv_query_kset = kset_create_and_add("query", NULL, uv_kobj);
664 if (!uv_query_kset) {
665 rc = -ENOMEM;
666 goto out_ind_files;
667 }
668
669 rc = sysfs_create_group(&uv_query_kset->kobj, &uv_query_attr_group);
670 if (!rc)
671 return 0;
672
673 kset_unregister(uv_query_kset);
674out_ind_files:
675 sysfs_remove_files(uv_kobj, uv_prot_virt_attrs);
676out_kobj:
677 kobject_del(uv_kobj);
678 kobject_put(uv_kobj);
679 return rc;
680}
681device_initcall(uv_info_init);
682#endif