Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Common Ultravisor functions and initialization
4 *
5 * Copyright IBM Corp. 2019, 2020
6 */
7#define KMSG_COMPONENT "prot_virt"
8#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
9
10#include <linux/kernel.h>
11#include <linux/types.h>
12#include <linux/sizes.h>
13#include <linux/bitmap.h>
14#include <linux/memblock.h>
15#include <linux/pagemap.h>
16#include <linux/swap.h>
17#include <asm/facility.h>
18#include <asm/sections.h>
19#include <asm/uv.h>
20
21/* the bootdata_preserved fields come from ones in arch/s390/boot/uv.c */
22#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
23int __bootdata_preserved(prot_virt_guest);
24#endif
25
26struct uv_info __bootdata_preserved(uv_info);
27
28#if IS_ENABLED(CONFIG_KVM)
29int __bootdata_preserved(prot_virt_host);
30EXPORT_SYMBOL(prot_virt_host);
31EXPORT_SYMBOL(uv_info);
32
33static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
34{
35 struct uv_cb_init uvcb = {
36 .header.cmd = UVC_CMD_INIT_UV,
37 .header.len = sizeof(uvcb),
38 .stor_origin = stor_base,
39 .stor_len = stor_len,
40 };
41
42 if (uv_call(0, (uint64_t)&uvcb)) {
43 pr_err("Ultravisor init failed with rc: 0x%x rrc: 0%x\n",
44 uvcb.header.rc, uvcb.header.rrc);
45 return -1;
46 }
47 return 0;
48}
49
50void __init setup_uv(void)
51{
52 unsigned long uv_stor_base;
53
54 /*
55 * keep these conditions in line with has_uv_sec_stor_limit()
56 */
57 if (!is_prot_virt_host())
58 return;
59
60 if (is_prot_virt_guest()) {
61 prot_virt_host = 0;
62 pr_warn("Protected virtualization not available in protected guests.");
63 return;
64 }
65
66 if (!test_facility(158)) {
67 prot_virt_host = 0;
68 pr_warn("Protected virtualization not supported by the hardware.");
69 return;
70 }
71
72 uv_stor_base = (unsigned long)memblock_alloc_try_nid(
73 uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
74 MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
75 if (!uv_stor_base) {
76 pr_warn("Failed to reserve %lu bytes for ultravisor base storage\n",
77 uv_info.uv_base_stor_len);
78 goto fail;
79 }
80
81 if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
82 memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
83 goto fail;
84 }
85
86 pr_info("Reserving %luMB as ultravisor base storage\n",
87 uv_info.uv_base_stor_len >> 20);
88 return;
89fail:
90 pr_info("Disabling support for protected virtualization");
91 prot_virt_host = 0;
92}
93
94/*
95 * Requests the Ultravisor to pin the page in the shared state. This will
96 * cause an intercept when the guest attempts to unshare the pinned page.
97 */
98static int uv_pin_shared(unsigned long paddr)
99{
100 struct uv_cb_cfs uvcb = {
101 .header.cmd = UVC_CMD_PIN_PAGE_SHARED,
102 .header.len = sizeof(uvcb),
103 .paddr = paddr,
104 };
105
106 if (uv_call(0, (u64)&uvcb))
107 return -EINVAL;
108 return 0;
109}
110
111/*
112 * Requests the Ultravisor to destroy a guest page and make it
113 * accessible to the host. The destroy clears the page instead of
114 * exporting.
115 *
116 * @paddr: Absolute host address of page to be destroyed
117 */
118int uv_destroy_page(unsigned long paddr)
119{
120 struct uv_cb_cfs uvcb = {
121 .header.cmd = UVC_CMD_DESTR_SEC_STOR,
122 .header.len = sizeof(uvcb),
123 .paddr = paddr
124 };
125
126 if (uv_call(0, (u64)&uvcb)) {
127 /*
128 * Older firmware uses 107/d as an indication of a non secure
129 * page. Let us emulate the newer variant (no-op).
130 */
131 if (uvcb.header.rc == 0x107 && uvcb.header.rrc == 0xd)
132 return 0;
133 return -EINVAL;
134 }
135 return 0;
136}
137
138/*
139 * Requests the Ultravisor to encrypt a guest page and make it
140 * accessible to the host for paging (export).
141 *
142 * @paddr: Absolute host address of page to be exported
143 */
144int uv_convert_from_secure(unsigned long paddr)
145{
146 struct uv_cb_cfs uvcb = {
147 .header.cmd = UVC_CMD_CONV_FROM_SEC_STOR,
148 .header.len = sizeof(uvcb),
149 .paddr = paddr
150 };
151
152 if (uv_call(0, (u64)&uvcb))
153 return -EINVAL;
154 return 0;
155}
156
157/*
158 * Calculate the expected ref_count for a page that would otherwise have no
159 * further pins. This was cribbed from similar functions in other places in
160 * the kernel, but with some slight modifications. We know that a secure
161 * page can not be a huge page for example.
162 */
163static int expected_page_refs(struct page *page)
164{
165 int res;
166
167 res = page_mapcount(page);
168 if (PageSwapCache(page)) {
169 res++;
170 } else if (page_mapping(page)) {
171 res++;
172 if (page_has_private(page))
173 res++;
174 }
175 return res;
176}
177
178static int make_secure_pte(pte_t *ptep, unsigned long addr,
179 struct page *exp_page, struct uv_cb_header *uvcb)
180{
181 pte_t entry = READ_ONCE(*ptep);
182 struct page *page;
183 int expected, rc = 0;
184
185 if (!pte_present(entry))
186 return -ENXIO;
187 if (pte_val(entry) & _PAGE_INVALID)
188 return -ENXIO;
189
190 page = pte_page(entry);
191 if (page != exp_page)
192 return -ENXIO;
193 if (PageWriteback(page))
194 return -EAGAIN;
195 expected = expected_page_refs(page);
196 if (!page_ref_freeze(page, expected))
197 return -EBUSY;
198 set_bit(PG_arch_1, &page->flags);
199 rc = uv_call(0, (u64)uvcb);
200 page_ref_unfreeze(page, expected);
201 /* Return -ENXIO if the page was not mapped, -EINVAL otherwise */
202 if (rc)
203 rc = uvcb->rc == 0x10a ? -ENXIO : -EINVAL;
204 return rc;
205}
206
207/*
208 * Requests the Ultravisor to make a page accessible to a guest.
209 * If it's brought in the first time, it will be cleared. If
210 * it has been exported before, it will be decrypted and integrity
211 * checked.
212 */
213int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb)
214{
215 struct vm_area_struct *vma;
216 bool local_drain = false;
217 spinlock_t *ptelock;
218 unsigned long uaddr;
219 struct page *page;
220 pte_t *ptep;
221 int rc;
222
223again:
224 rc = -EFAULT;
225 mmap_read_lock(gmap->mm);
226
227 uaddr = __gmap_translate(gmap, gaddr);
228 if (IS_ERR_VALUE(uaddr))
229 goto out;
230 vma = find_vma(gmap->mm, uaddr);
231 if (!vma)
232 goto out;
233 /*
234 * Secure pages cannot be huge and userspace should not combine both.
235 * In case userspace does it anyway this will result in an -EFAULT for
236 * the unpack. The guest is thus never reaching secure mode. If
237 * userspace is playing dirty tricky with mapping huge pages later
238 * on this will result in a segmentation fault.
239 */
240 if (is_vm_hugetlb_page(vma))
241 goto out;
242
243 rc = -ENXIO;
244 page = follow_page(vma, uaddr, FOLL_WRITE);
245 if (IS_ERR_OR_NULL(page))
246 goto out;
247
248 lock_page(page);
249 ptep = get_locked_pte(gmap->mm, uaddr, &ptelock);
250 rc = make_secure_pte(ptep, uaddr, page, uvcb);
251 pte_unmap_unlock(ptep, ptelock);
252 unlock_page(page);
253out:
254 mmap_read_unlock(gmap->mm);
255
256 if (rc == -EAGAIN) {
257 wait_on_page_writeback(page);
258 } else if (rc == -EBUSY) {
259 /*
260 * If we have tried a local drain and the page refcount
261 * still does not match our expected safe value, try with a
262 * system wide drain. This is needed if the pagevecs holding
263 * the page are on a different CPU.
264 */
265 if (local_drain) {
266 lru_add_drain_all();
267 /* We give up here, and let the caller try again */
268 return -EAGAIN;
269 }
270 /*
271 * We are here if the page refcount does not match the
272 * expected safe value. The main culprits are usually
273 * pagevecs. With lru_add_drain() we drain the pagevecs
274 * on the local CPU so that hopefully the refcount will
275 * reach the expected safe value.
276 */
277 lru_add_drain();
278 local_drain = true;
279 /* And now we try again immediately after draining */
280 goto again;
281 } else if (rc == -ENXIO) {
282 if (gmap_fault(gmap, gaddr, FAULT_FLAG_WRITE))
283 return -EFAULT;
284 return -EAGAIN;
285 }
286 return rc;
287}
288EXPORT_SYMBOL_GPL(gmap_make_secure);
289
290int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr)
291{
292 struct uv_cb_cts uvcb = {
293 .header.cmd = UVC_CMD_CONV_TO_SEC_STOR,
294 .header.len = sizeof(uvcb),
295 .guest_handle = gmap->guest_handle,
296 .gaddr = gaddr,
297 };
298
299 return gmap_make_secure(gmap, gaddr, &uvcb);
300}
301EXPORT_SYMBOL_GPL(gmap_convert_to_secure);
302
303/*
304 * To be called with the page locked or with an extra reference! This will
305 * prevent gmap_make_secure from touching the page concurrently. Having 2
306 * parallel make_page_accessible is fine, as the UV calls will become a
307 * no-op if the page is already exported.
308 */
309int arch_make_page_accessible(struct page *page)
310{
311 int rc = 0;
312
313 /* Hugepage cannot be protected, so nothing to do */
314 if (PageHuge(page))
315 return 0;
316
317 /*
318 * PG_arch_1 is used in 3 places:
319 * 1. for kernel page tables during early boot
320 * 2. for storage keys of huge pages and KVM
321 * 3. As an indication that this page might be secure. This can
322 * overindicate, e.g. we set the bit before calling
323 * convert_to_secure.
324 * As secure pages are never huge, all 3 variants can co-exists.
325 */
326 if (!test_bit(PG_arch_1, &page->flags))
327 return 0;
328
329 rc = uv_pin_shared(page_to_phys(page));
330 if (!rc) {
331 clear_bit(PG_arch_1, &page->flags);
332 return 0;
333 }
334
335 rc = uv_convert_from_secure(page_to_phys(page));
336 if (!rc) {
337 clear_bit(PG_arch_1, &page->flags);
338 return 0;
339 }
340
341 return rc;
342}
343EXPORT_SYMBOL_GPL(arch_make_page_accessible);
344
345#endif
346
347#if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM)
348static ssize_t uv_query_facilities(struct kobject *kobj,
349 struct kobj_attribute *attr, char *page)
350{
351 return scnprintf(page, PAGE_SIZE, "%lx\n%lx\n%lx\n%lx\n",
352 uv_info.inst_calls_list[0],
353 uv_info.inst_calls_list[1],
354 uv_info.inst_calls_list[2],
355 uv_info.inst_calls_list[3]);
356}
357
358static struct kobj_attribute uv_query_facilities_attr =
359 __ATTR(facilities, 0444, uv_query_facilities, NULL);
360
361static ssize_t uv_query_feature_indications(struct kobject *kobj,
362 struct kobj_attribute *attr, char *buf)
363{
364 return sysfs_emit(buf, "%lx\n", uv_info.uv_feature_indications);
365}
366
367static struct kobj_attribute uv_query_feature_indications_attr =
368 __ATTR(feature_indications, 0444, uv_query_feature_indications, NULL);
369
370static ssize_t uv_query_max_guest_cpus(struct kobject *kobj,
371 struct kobj_attribute *attr, char *page)
372{
373 return scnprintf(page, PAGE_SIZE, "%d\n",
374 uv_info.max_guest_cpu_id + 1);
375}
376
377static struct kobj_attribute uv_query_max_guest_cpus_attr =
378 __ATTR(max_cpus, 0444, uv_query_max_guest_cpus, NULL);
379
380static ssize_t uv_query_max_guest_vms(struct kobject *kobj,
381 struct kobj_attribute *attr, char *page)
382{
383 return scnprintf(page, PAGE_SIZE, "%d\n",
384 uv_info.max_num_sec_conf);
385}
386
387static struct kobj_attribute uv_query_max_guest_vms_attr =
388 __ATTR(max_guests, 0444, uv_query_max_guest_vms, NULL);
389
390static ssize_t uv_query_max_guest_addr(struct kobject *kobj,
391 struct kobj_attribute *attr, char *page)
392{
393 return scnprintf(page, PAGE_SIZE, "%lx\n",
394 uv_info.max_sec_stor_addr);
395}
396
397static struct kobj_attribute uv_query_max_guest_addr_attr =
398 __ATTR(max_address, 0444, uv_query_max_guest_addr, NULL);
399
400static struct attribute *uv_query_attrs[] = {
401 &uv_query_facilities_attr.attr,
402 &uv_query_feature_indications_attr.attr,
403 &uv_query_max_guest_cpus_attr.attr,
404 &uv_query_max_guest_vms_attr.attr,
405 &uv_query_max_guest_addr_attr.attr,
406 NULL,
407};
408
409static struct attribute_group uv_query_attr_group = {
410 .attrs = uv_query_attrs,
411};
412
413static ssize_t uv_is_prot_virt_guest(struct kobject *kobj,
414 struct kobj_attribute *attr, char *page)
415{
416 int val = 0;
417
418#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
419 val = prot_virt_guest;
420#endif
421 return scnprintf(page, PAGE_SIZE, "%d\n", val);
422}
423
424static ssize_t uv_is_prot_virt_host(struct kobject *kobj,
425 struct kobj_attribute *attr, char *page)
426{
427 int val = 0;
428
429#if IS_ENABLED(CONFIG_KVM)
430 val = prot_virt_host;
431#endif
432
433 return scnprintf(page, PAGE_SIZE, "%d\n", val);
434}
435
436static struct kobj_attribute uv_prot_virt_guest =
437 __ATTR(prot_virt_guest, 0444, uv_is_prot_virt_guest, NULL);
438
439static struct kobj_attribute uv_prot_virt_host =
440 __ATTR(prot_virt_host, 0444, uv_is_prot_virt_host, NULL);
441
442static const struct attribute *uv_prot_virt_attrs[] = {
443 &uv_prot_virt_guest.attr,
444 &uv_prot_virt_host.attr,
445 NULL,
446};
447
448static struct kset *uv_query_kset;
449static struct kobject *uv_kobj;
450
451static int __init uv_info_init(void)
452{
453 int rc = -ENOMEM;
454
455 if (!test_facility(158))
456 return 0;
457
458 uv_kobj = kobject_create_and_add("uv", firmware_kobj);
459 if (!uv_kobj)
460 return -ENOMEM;
461
462 rc = sysfs_create_files(uv_kobj, uv_prot_virt_attrs);
463 if (rc)
464 goto out_kobj;
465
466 uv_query_kset = kset_create_and_add("query", NULL, uv_kobj);
467 if (!uv_query_kset) {
468 rc = -ENOMEM;
469 goto out_ind_files;
470 }
471
472 rc = sysfs_create_group(&uv_query_kset->kobj, &uv_query_attr_group);
473 if (!rc)
474 return 0;
475
476 kset_unregister(uv_query_kset);
477out_ind_files:
478 sysfs_remove_files(uv_kobj, uv_prot_virt_attrs);
479out_kobj:
480 kobject_del(uv_kobj);
481 kobject_put(uv_kobj);
482 return rc;
483}
484device_initcall(uv_info_init);
485#endif