Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 */
7
8/*
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
11 * compilation.
12 */
13#include "misc.h"
14
15#include <asm/bootparam.h>
16#include <asm/pgtable_types.h>
17#include <asm/sev.h>
18#include <asm/trapnr.h>
19#include <asm/trap_pf.h>
20#include <asm/msr-index.h>
21#include <asm/fpu/xcr.h>
22#include <asm/ptrace.h>
23#include <asm/svm.h>
24#include <asm/cpuid.h>
25
26#include "error.h"
27#include "../msr.h"
28
29static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
30struct ghcb *boot_ghcb;
31
32/*
33 * Copy a version of this function here - insn-eval.c can't be used in
34 * pre-decompression code.
35 */
36static bool insn_has_rep_prefix(struct insn *insn)
37{
38 insn_byte_t p;
39 int i;
40
41 insn_get_prefixes(insn);
42
43 for_each_insn_prefix(insn, i, p) {
44 if (p == 0xf2 || p == 0xf3)
45 return true;
46 }
47
48 return false;
49}
50
51/*
52 * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
53 * doesn't use segments.
54 */
55static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
56{
57 return 0UL;
58}
59
60static inline u64 sev_es_rd_ghcb_msr(void)
61{
62 struct msr m;
63
64 boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
65
66 return m.q;
67}
68
69static inline void sev_es_wr_ghcb_msr(u64 val)
70{
71 struct msr m;
72
73 m.q = val;
74 boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
75}
76
77static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
78{
79 char buffer[MAX_INSN_SIZE];
80 int ret;
81
82 memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
83
84 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
85 if (ret < 0)
86 return ES_DECODE_FAILED;
87
88 return ES_OK;
89}
90
91static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
92 void *dst, char *buf, size_t size)
93{
94 memcpy(dst, buf, size);
95
96 return ES_OK;
97}
98
99static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
100 void *src, char *buf, size_t size)
101{
102 memcpy(buf, src, size);
103
104 return ES_OK;
105}
106
107static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
108{
109 return ES_OK;
110}
111
112static bool fault_in_kernel_space(unsigned long address)
113{
114 return false;
115}
116
117#undef __init
118#define __init
119
120#undef __head
121#define __head
122
123#define __BOOT_COMPRESSED
124
125/* Basic instruction decoding support needed */
126#include "../../lib/inat.c"
127#include "../../lib/insn.c"
128
129/* Include code for early handlers */
130#include "../../coco/sev/shared.c"
131
132static struct svsm_ca *svsm_get_caa(void)
133{
134 return boot_svsm_caa;
135}
136
137static u64 svsm_get_caa_pa(void)
138{
139 return boot_svsm_caa_pa;
140}
141
142static int svsm_perform_call_protocol(struct svsm_call *call)
143{
144 struct ghcb *ghcb;
145 int ret;
146
147 if (boot_ghcb)
148 ghcb = boot_ghcb;
149 else
150 ghcb = NULL;
151
152 do {
153 ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
154 : svsm_perform_msr_protocol(call);
155 } while (ret == -EAGAIN);
156
157 return ret;
158}
159
160bool sev_snp_enabled(void)
161{
162 return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
163}
164
165static void __page_state_change(unsigned long paddr, enum psc_op op)
166{
167 u64 val;
168
169 if (!sev_snp_enabled())
170 return;
171
172 /*
173 * If private -> shared then invalidate the page before requesting the
174 * state change in the RMP table.
175 */
176 if (op == SNP_PAGE_STATE_SHARED)
177 pvalidate_4k_page(paddr, paddr, false);
178
179 /* Issue VMGEXIT to change the page state in RMP table. */
180 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
181 VMGEXIT();
182
183 /* Read the response of the VMGEXIT. */
184 val = sev_es_rd_ghcb_msr();
185 if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
186 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
187
188 /*
189 * Now that page state is changed in the RMP table, validate it so that it is
190 * consistent with the RMP entry.
191 */
192 if (op == SNP_PAGE_STATE_PRIVATE)
193 pvalidate_4k_page(paddr, paddr, true);
194}
195
196void snp_set_page_private(unsigned long paddr)
197{
198 __page_state_change(paddr, SNP_PAGE_STATE_PRIVATE);
199}
200
201void snp_set_page_shared(unsigned long paddr)
202{
203 __page_state_change(paddr, SNP_PAGE_STATE_SHARED);
204}
205
206static bool early_setup_ghcb(void)
207{
208 if (set_page_decrypted((unsigned long)&boot_ghcb_page))
209 return false;
210
211 /* Page is now mapped decrypted, clear it */
212 memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page));
213
214 boot_ghcb = &boot_ghcb_page;
215
216 /* Initialize lookup tables for the instruction decoder */
217 inat_init_tables();
218
219 /* SNP guest requires the GHCB GPA must be registered */
220 if (sev_snp_enabled())
221 snp_register_ghcb_early(__pa(&boot_ghcb_page));
222
223 return true;
224}
225
226static phys_addr_t __snp_accept_memory(struct snp_psc_desc *desc,
227 phys_addr_t pa, phys_addr_t pa_end)
228{
229 struct psc_hdr *hdr;
230 struct psc_entry *e;
231 unsigned int i;
232
233 hdr = &desc->hdr;
234 memset(hdr, 0, sizeof(*hdr));
235
236 e = desc->entries;
237
238 i = 0;
239 while (pa < pa_end && i < VMGEXIT_PSC_MAX_ENTRY) {
240 hdr->end_entry = i;
241
242 e->gfn = pa >> PAGE_SHIFT;
243 e->operation = SNP_PAGE_STATE_PRIVATE;
244 if (IS_ALIGNED(pa, PMD_SIZE) && (pa_end - pa) >= PMD_SIZE) {
245 e->pagesize = RMP_PG_SIZE_2M;
246 pa += PMD_SIZE;
247 } else {
248 e->pagesize = RMP_PG_SIZE_4K;
249 pa += PAGE_SIZE;
250 }
251
252 e++;
253 i++;
254 }
255
256 if (vmgexit_psc(boot_ghcb, desc))
257 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
258
259 pvalidate_pages(desc);
260
261 return pa;
262}
263
264void snp_accept_memory(phys_addr_t start, phys_addr_t end)
265{
266 struct snp_psc_desc desc = {};
267 unsigned int i;
268 phys_addr_t pa;
269
270 if (!boot_ghcb && !early_setup_ghcb())
271 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
272
273 pa = start;
274 while (pa < end)
275 pa = __snp_accept_memory(&desc, pa, end);
276}
277
278void sev_es_shutdown_ghcb(void)
279{
280 if (!boot_ghcb)
281 return;
282
283 if (!sev_es_check_cpu_features())
284 error("SEV-ES CPU Features missing.");
285
286 /*
287 * This denotes whether to use the GHCB MSR protocol or the GHCB
288 * shared page to perform a GHCB request. Since the GHCB page is
289 * being changed to encrypted, it can't be used to perform GHCB
290 * requests. Clear the boot_ghcb variable so that the GHCB MSR
291 * protocol is used to change the GHCB page over to an encrypted
292 * page.
293 */
294 boot_ghcb = NULL;
295
296 /*
297 * GHCB Page must be flushed from the cache and mapped encrypted again.
298 * Otherwise the running kernel will see strange cache effects when
299 * trying to use that page.
300 */
301 if (set_page_encrypted((unsigned long)&boot_ghcb_page))
302 error("Can't map GHCB page encrypted");
303
304 /*
305 * GHCB page is mapped encrypted again and flushed from the cache.
306 * Mark it non-present now to catch bugs when #VC exceptions trigger
307 * after this point.
308 */
309 if (set_page_non_present((unsigned long)&boot_ghcb_page))
310 error("Can't unmap GHCB page");
311}
312
313static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set,
314 unsigned int reason, u64 exit_info_2)
315{
316 u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason);
317
318 vc_ghcb_invalidate(ghcb);
319 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST);
320 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
321 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
322
323 sev_es_wr_ghcb_msr(__pa(ghcb));
324 VMGEXIT();
325
326 while (true)
327 asm volatile("hlt\n" : : : "memory");
328}
329
330bool sev_es_check_ghcb_fault(unsigned long address)
331{
332 /* Check whether the fault was on the GHCB page */
333 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
334}
335
336void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
337{
338 struct es_em_ctxt ctxt;
339 enum es_result result;
340
341 if (!boot_ghcb && !early_setup_ghcb())
342 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
343
344 vc_ghcb_invalidate(boot_ghcb);
345 result = vc_init_em_ctxt(&ctxt, regs, exit_code);
346 if (result != ES_OK)
347 goto finish;
348
349 result = vc_check_opcode_bytes(&ctxt, exit_code);
350 if (result != ES_OK)
351 goto finish;
352
353 switch (exit_code) {
354 case SVM_EXIT_RDTSC:
355 case SVM_EXIT_RDTSCP:
356 result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
357 break;
358 case SVM_EXIT_IOIO:
359 result = vc_handle_ioio(boot_ghcb, &ctxt);
360 break;
361 case SVM_EXIT_CPUID:
362 result = vc_handle_cpuid(boot_ghcb, &ctxt);
363 break;
364 default:
365 result = ES_UNSUPPORTED;
366 break;
367 }
368
369finish:
370 if (result == ES_OK)
371 vc_finish_insn(&ctxt);
372 else if (result != ES_RETRY)
373 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
374}
375
376/*
377 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
378 * guest side implementation for proper functioning of the guest. If any
379 * of these features are enabled in the hypervisor but are lacking guest
380 * side implementation, the behavior of the guest will be undefined. The
381 * guest could fail in non-obvious way making it difficult to debug.
382 *
383 * As the behavior of reserved feature bits is unknown to be on the
384 * safe side add them to the required features mask.
385 */
386#define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \
387 MSR_AMD64_SNP_REFLECT_VC | \
388 MSR_AMD64_SNP_RESTRICTED_INJ | \
389 MSR_AMD64_SNP_ALT_INJ | \
390 MSR_AMD64_SNP_DEBUG_SWAP | \
391 MSR_AMD64_SNP_VMPL_SSS | \
392 MSR_AMD64_SNP_SECURE_TSC | \
393 MSR_AMD64_SNP_VMGEXIT_PARAM | \
394 MSR_AMD64_SNP_VMSA_REG_PROT | \
395 MSR_AMD64_SNP_RESERVED_BIT13 | \
396 MSR_AMD64_SNP_RESERVED_BIT15 | \
397 MSR_AMD64_SNP_RESERVED_MASK)
398
399/*
400 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
401 * by the guest kernel. As and when a new feature is implemented in the
402 * guest kernel, a corresponding bit should be added to the mask.
403 */
404#define SNP_FEATURES_PRESENT MSR_AMD64_SNP_DEBUG_SWAP
405
406u64 snp_get_unsupported_features(u64 status)
407{
408 if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
409 return 0;
410
411 return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
412}
413
414void snp_check_features(void)
415{
416 u64 unsupported;
417
418 /*
419 * Terminate the boot if hypervisor has enabled any feature lacking
420 * guest side implementation. Pass on the unsupported features mask through
421 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
422 * as part of the guest boot failure.
423 */
424 unsupported = snp_get_unsupported_features(sev_status);
425 if (unsupported) {
426 if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb()))
427 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
428
429 sev_es_ghcb_terminate(boot_ghcb, SEV_TERM_SET_GEN,
430 GHCB_SNP_UNSUPPORTED, unsupported);
431 }
432}
433
434/* Search for Confidential Computing blob in the EFI config table. */
435static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
436{
437 unsigned long cfg_table_pa;
438 unsigned int cfg_table_len;
439 int ret;
440
441 ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len);
442 if (ret)
443 return NULL;
444
445 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa,
446 cfg_table_len,
447 EFI_CC_BLOB_GUID);
448}
449
450/*
451 * Initial set up of SNP relies on information provided by the
452 * Confidential Computing blob, which can be passed to the boot kernel
453 * by firmware/bootloader in the following ways:
454 *
455 * - via an entry in the EFI config table
456 * - via a setup_data structure, as defined by the Linux Boot Protocol
457 *
458 * Scan for the blob in that order.
459 */
460static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
461{
462 struct cc_blob_sev_info *cc_info;
463
464 cc_info = find_cc_blob_efi(bp);
465 if (cc_info)
466 goto found_cc_info;
467
468 cc_info = find_cc_blob_setup_data(bp);
469 if (!cc_info)
470 return NULL;
471
472found_cc_info:
473 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
474 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
475
476 return cc_info;
477}
478
479/*
480 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
481 * will verify the SNP CPUID/MSR bits.
482 */
483static bool early_snp_init(struct boot_params *bp)
484{
485 struct cc_blob_sev_info *cc_info;
486
487 if (!bp)
488 return false;
489
490 cc_info = find_cc_blob(bp);
491 if (!cc_info)
492 return false;
493
494 /*
495 * If a SNP-specific Confidential Computing blob is present, then
496 * firmware/bootloader have indicated SNP support. Verifying this
497 * involves CPUID checks which will be more reliable if the SNP
498 * CPUID table is used. See comments over snp_setup_cpuid_table() for
499 * more details.
500 */
501 setup_cpuid_table(cc_info);
502
503 /*
504 * Record the SVSM Calling Area (CA) address if the guest is not
505 * running at VMPL0. The CA will be used to communicate with the
506 * SVSM and request its services.
507 */
508 svsm_setup_ca(cc_info);
509
510 /*
511 * Pass run-time kernel a pointer to CC info via boot_params so EFI
512 * config table doesn't need to be searched again during early startup
513 * phase.
514 */
515 bp->cc_blob_address = (u32)(unsigned long)cc_info;
516
517 return true;
518}
519
520/*
521 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
522 *
523 * Returns < 0 if SEV is not supported, otherwise the position of the
524 * encryption bit in the page table descriptors.
525 */
526static int sev_check_cpu_support(void)
527{
528 unsigned int eax, ebx, ecx, edx;
529
530 /* Check for the SME/SEV support leaf */
531 eax = 0x80000000;
532 ecx = 0;
533 native_cpuid(&eax, &ebx, &ecx, &edx);
534 if (eax < 0x8000001f)
535 return -ENODEV;
536
537 /*
538 * Check for the SME/SEV feature:
539 * CPUID Fn8000_001F[EAX]
540 * - Bit 0 - Secure Memory Encryption support
541 * - Bit 1 - Secure Encrypted Virtualization support
542 * CPUID Fn8000_001F[EBX]
543 * - Bits 5:0 - Pagetable bit position used to indicate encryption
544 */
545 eax = 0x8000001f;
546 ecx = 0;
547 native_cpuid(&eax, &ebx, &ecx, &edx);
548 /* Check whether SEV is supported */
549 if (!(eax & BIT(1)))
550 return -ENODEV;
551
552 return ebx & 0x3f;
553}
554
555void sev_enable(struct boot_params *bp)
556{
557 struct msr m;
558 int bitpos;
559 bool snp;
560
561 /*
562 * bp->cc_blob_address should only be set by boot/compressed kernel.
563 * Initialize it to 0 to ensure that uninitialized values from
564 * buggy bootloaders aren't propagated.
565 */
566 if (bp)
567 bp->cc_blob_address = 0;
568
569 /*
570 * Do an initial SEV capability check before early_snp_init() which
571 * loads the CPUID page and the same checks afterwards are done
572 * without the hypervisor and are trustworthy.
573 *
574 * If the HV fakes SEV support, the guest will crash'n'burn
575 * which is good enough.
576 */
577
578 if (sev_check_cpu_support() < 0)
579 return;
580
581 /*
582 * Setup/preliminary detection of SNP. This will be sanity-checked
583 * against CPUID/MSR values later.
584 */
585 snp = early_snp_init(bp);
586
587 /* Now repeat the checks with the SNP CPUID table. */
588
589 bitpos = sev_check_cpu_support();
590 if (bitpos < 0) {
591 if (snp)
592 error("SEV-SNP support indicated by CC blob, but not CPUID.");
593 return;
594 }
595
596 /* Set the SME mask if this is an SEV guest. */
597 boot_rdmsr(MSR_AMD64_SEV, &m);
598 sev_status = m.q;
599 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
600 return;
601
602 /* Negotiate the GHCB protocol version. */
603 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
604 if (!sev_es_negotiate_protocol())
605 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
606 }
607
608 /*
609 * SNP is supported in v2 of the GHCB spec which mandates support for HV
610 * features.
611 */
612 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
613 u64 hv_features;
614 int ret;
615
616 hv_features = get_hv_features();
617 if (!(hv_features & GHCB_HV_FT_SNP))
618 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
619
620 /*
621 * Enforce running at VMPL0 or with an SVSM.
622 *
623 * Use RMPADJUST (see the rmpadjust() function for a description of
624 * what the instruction does) to update the VMPL1 permissions of a
625 * page. If the guest is running at VMPL0, this will succeed. If the
626 * guest is running at any other VMPL, this will fail. Linux SNP guests
627 * only ever run at a single VMPL level so permission mask changes of a
628 * lesser-privileged VMPL are a don't-care.
629 */
630 ret = rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, 1);
631
632 /*
633 * Running at VMPL0 is not required if an SVSM is present and the hypervisor
634 * supports the required SVSM GHCB events.
635 */
636 if (ret &&
637 !(snp_vmpl && (hv_features & GHCB_HV_FT_SNP_MULTI_VMPL)))
638 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
639 }
640
641 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
642 error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
643
644 sme_me_mask = BIT_ULL(bitpos);
645}
646
647/*
648 * sev_get_status - Retrieve the SEV status mask
649 *
650 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
651 * AMD64_SEV MSR.
652 */
653u64 sev_get_status(void)
654{
655 struct msr m;
656
657 if (sev_check_cpu_support() < 0)
658 return 0;
659
660 boot_rdmsr(MSR_AMD64_SEV, &m);
661 return m.q;
662}
663
664void sev_prep_identity_maps(unsigned long top_level_pgt)
665{
666 /*
667 * The Confidential Computing blob is used very early in uncompressed
668 * kernel to find the in-memory CPUID table to handle CPUID
669 * instructions. Make sure an identity-mapping exists so it can be
670 * accessed after switchover.
671 */
672 if (sev_snp_enabled()) {
673 unsigned long cc_info_pa = boot_params_ptr->cc_blob_address;
674 struct cc_blob_sev_info *cc_info;
675
676 kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info));
677
678 cc_info = (struct cc_blob_sev_info *)cc_info_pa;
679 kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len);
680 }
681
682 sev_verify_cbit(top_level_pgt);
683}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 */
7
8/*
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
11 * compilation.
12 */
13#include "misc.h"
14
15#include <asm/pgtable_types.h>
16#include <asm/sev.h>
17#include <asm/trapnr.h>
18#include <asm/trap_pf.h>
19#include <asm/msr-index.h>
20#include <asm/fpu/xcr.h>
21#include <asm/ptrace.h>
22#include <asm/svm.h>
23#include <asm/cpuid.h>
24
25#include "error.h"
26#include "../msr.h"
27
28static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
29struct ghcb *boot_ghcb;
30
31/*
32 * Copy a version of this function here - insn-eval.c can't be used in
33 * pre-decompression code.
34 */
35static bool insn_has_rep_prefix(struct insn *insn)
36{
37 insn_byte_t p;
38 int i;
39
40 insn_get_prefixes(insn);
41
42 for_each_insn_prefix(insn, i, p) {
43 if (p == 0xf2 || p == 0xf3)
44 return true;
45 }
46
47 return false;
48}
49
50/*
51 * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
52 * doesn't use segments.
53 */
54static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
55{
56 return 0UL;
57}
58
59static inline u64 sev_es_rd_ghcb_msr(void)
60{
61 struct msr m;
62
63 boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
64
65 return m.q;
66}
67
68static inline void sev_es_wr_ghcb_msr(u64 val)
69{
70 struct msr m;
71
72 m.q = val;
73 boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
74}
75
76static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
77{
78 char buffer[MAX_INSN_SIZE];
79 int ret;
80
81 memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
82
83 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
84 if (ret < 0)
85 return ES_DECODE_FAILED;
86
87 return ES_OK;
88}
89
90static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
91 void *dst, char *buf, size_t size)
92{
93 memcpy(dst, buf, size);
94
95 return ES_OK;
96}
97
98static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
99 void *src, char *buf, size_t size)
100{
101 memcpy(buf, src, size);
102
103 return ES_OK;
104}
105
106static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
107{
108 return ES_OK;
109}
110
111static bool fault_in_kernel_space(unsigned long address)
112{
113 return false;
114}
115
116#undef __init
117#define __init
118
119#define __BOOT_COMPRESSED
120
121/* Basic instruction decoding support needed */
122#include "../../lib/inat.c"
123#include "../../lib/insn.c"
124
125/* Include code for early handlers */
126#include "../../kernel/sev-shared.c"
127
128bool sev_snp_enabled(void)
129{
130 return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
131}
132
133static void __page_state_change(unsigned long paddr, enum psc_op op)
134{
135 u64 val;
136
137 if (!sev_snp_enabled())
138 return;
139
140 /*
141 * If private -> shared then invalidate the page before requesting the
142 * state change in the RMP table.
143 */
144 if (op == SNP_PAGE_STATE_SHARED && pvalidate(paddr, RMP_PG_SIZE_4K, 0))
145 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
146
147 /* Issue VMGEXIT to change the page state in RMP table. */
148 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
149 VMGEXIT();
150
151 /* Read the response of the VMGEXIT. */
152 val = sev_es_rd_ghcb_msr();
153 if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
154 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
155
156 /*
157 * Now that page state is changed in the RMP table, validate it so that it is
158 * consistent with the RMP entry.
159 */
160 if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(paddr, RMP_PG_SIZE_4K, 1))
161 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
162}
163
164void snp_set_page_private(unsigned long paddr)
165{
166 __page_state_change(paddr, SNP_PAGE_STATE_PRIVATE);
167}
168
169void snp_set_page_shared(unsigned long paddr)
170{
171 __page_state_change(paddr, SNP_PAGE_STATE_SHARED);
172}
173
174static bool early_setup_ghcb(void)
175{
176 if (set_page_decrypted((unsigned long)&boot_ghcb_page))
177 return false;
178
179 /* Page is now mapped decrypted, clear it */
180 memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page));
181
182 boot_ghcb = &boot_ghcb_page;
183
184 /* Initialize lookup tables for the instruction decoder */
185 inat_init_tables();
186
187 /* SNP guest requires the GHCB GPA must be registered */
188 if (sev_snp_enabled())
189 snp_register_ghcb_early(__pa(&boot_ghcb_page));
190
191 return true;
192}
193
194static phys_addr_t __snp_accept_memory(struct snp_psc_desc *desc,
195 phys_addr_t pa, phys_addr_t pa_end)
196{
197 struct psc_hdr *hdr;
198 struct psc_entry *e;
199 unsigned int i;
200
201 hdr = &desc->hdr;
202 memset(hdr, 0, sizeof(*hdr));
203
204 e = desc->entries;
205
206 i = 0;
207 while (pa < pa_end && i < VMGEXIT_PSC_MAX_ENTRY) {
208 hdr->end_entry = i;
209
210 e->gfn = pa >> PAGE_SHIFT;
211 e->operation = SNP_PAGE_STATE_PRIVATE;
212 if (IS_ALIGNED(pa, PMD_SIZE) && (pa_end - pa) >= PMD_SIZE) {
213 e->pagesize = RMP_PG_SIZE_2M;
214 pa += PMD_SIZE;
215 } else {
216 e->pagesize = RMP_PG_SIZE_4K;
217 pa += PAGE_SIZE;
218 }
219
220 e++;
221 i++;
222 }
223
224 if (vmgexit_psc(boot_ghcb, desc))
225 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
226
227 pvalidate_pages(desc);
228
229 return pa;
230}
231
232void snp_accept_memory(phys_addr_t start, phys_addr_t end)
233{
234 struct snp_psc_desc desc = {};
235 unsigned int i;
236 phys_addr_t pa;
237
238 if (!boot_ghcb && !early_setup_ghcb())
239 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
240
241 pa = start;
242 while (pa < end)
243 pa = __snp_accept_memory(&desc, pa, end);
244}
245
246void sev_es_shutdown_ghcb(void)
247{
248 if (!boot_ghcb)
249 return;
250
251 if (!sev_es_check_cpu_features())
252 error("SEV-ES CPU Features missing.");
253
254 /*
255 * GHCB Page must be flushed from the cache and mapped encrypted again.
256 * Otherwise the running kernel will see strange cache effects when
257 * trying to use that page.
258 */
259 if (set_page_encrypted((unsigned long)&boot_ghcb_page))
260 error("Can't map GHCB page encrypted");
261
262 /*
263 * GHCB page is mapped encrypted again and flushed from the cache.
264 * Mark it non-present now to catch bugs when #VC exceptions trigger
265 * after this point.
266 */
267 if (set_page_non_present((unsigned long)&boot_ghcb_page))
268 error("Can't unmap GHCB page");
269}
270
271static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set,
272 unsigned int reason, u64 exit_info_2)
273{
274 u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason);
275
276 vc_ghcb_invalidate(ghcb);
277 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST);
278 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
279 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
280
281 sev_es_wr_ghcb_msr(__pa(ghcb));
282 VMGEXIT();
283
284 while (true)
285 asm volatile("hlt\n" : : : "memory");
286}
287
288bool sev_es_check_ghcb_fault(unsigned long address)
289{
290 /* Check whether the fault was on the GHCB page */
291 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
292}
293
294void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
295{
296 struct es_em_ctxt ctxt;
297 enum es_result result;
298
299 if (!boot_ghcb && !early_setup_ghcb())
300 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
301
302 vc_ghcb_invalidate(boot_ghcb);
303 result = vc_init_em_ctxt(&ctxt, regs, exit_code);
304 if (result != ES_OK)
305 goto finish;
306
307 switch (exit_code) {
308 case SVM_EXIT_RDTSC:
309 case SVM_EXIT_RDTSCP:
310 result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
311 break;
312 case SVM_EXIT_IOIO:
313 result = vc_handle_ioio(boot_ghcb, &ctxt);
314 break;
315 case SVM_EXIT_CPUID:
316 result = vc_handle_cpuid(boot_ghcb, &ctxt);
317 break;
318 default:
319 result = ES_UNSUPPORTED;
320 break;
321 }
322
323finish:
324 if (result == ES_OK)
325 vc_finish_insn(&ctxt);
326 else if (result != ES_RETRY)
327 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
328}
329
330static void enforce_vmpl0(void)
331{
332 u64 attrs;
333 int err;
334
335 /*
336 * RMPADJUST modifies RMP permissions of a lesser-privileged (numerically
337 * higher) privilege level. Here, clear the VMPL1 permission mask of the
338 * GHCB page. If the guest is not running at VMPL0, this will fail.
339 *
340 * If the guest is running at VMPL0, it will succeed. Even if that operation
341 * modifies permission bits, it is still ok to do so currently because Linux
342 * SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks
343 * changing is a don't-care.
344 */
345 attrs = 1;
346 if (rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs))
347 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
348}
349
350/*
351 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
352 * guest side implementation for proper functioning of the guest. If any
353 * of these features are enabled in the hypervisor but are lacking guest
354 * side implementation, the behavior of the guest will be undefined. The
355 * guest could fail in non-obvious way making it difficult to debug.
356 *
357 * As the behavior of reserved feature bits is unknown to be on the
358 * safe side add them to the required features mask.
359 */
360#define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \
361 MSR_AMD64_SNP_REFLECT_VC | \
362 MSR_AMD64_SNP_RESTRICTED_INJ | \
363 MSR_AMD64_SNP_ALT_INJ | \
364 MSR_AMD64_SNP_DEBUG_SWAP | \
365 MSR_AMD64_SNP_VMPL_SSS | \
366 MSR_AMD64_SNP_SECURE_TSC | \
367 MSR_AMD64_SNP_VMGEXIT_PARAM | \
368 MSR_AMD64_SNP_VMSA_REG_PROTECTION | \
369 MSR_AMD64_SNP_RESERVED_BIT13 | \
370 MSR_AMD64_SNP_RESERVED_BIT15 | \
371 MSR_AMD64_SNP_RESERVED_MASK)
372
373/*
374 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
375 * by the guest kernel. As and when a new feature is implemented in the
376 * guest kernel, a corresponding bit should be added to the mask.
377 */
378#define SNP_FEATURES_PRESENT MSR_AMD64_SNP_DEBUG_SWAP
379
380u64 snp_get_unsupported_features(u64 status)
381{
382 if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
383 return 0;
384
385 return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
386}
387
388void snp_check_features(void)
389{
390 u64 unsupported;
391
392 /*
393 * Terminate the boot if hypervisor has enabled any feature lacking
394 * guest side implementation. Pass on the unsupported features mask through
395 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
396 * as part of the guest boot failure.
397 */
398 unsupported = snp_get_unsupported_features(sev_status);
399 if (unsupported) {
400 if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb()))
401 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
402
403 sev_es_ghcb_terminate(boot_ghcb, SEV_TERM_SET_GEN,
404 GHCB_SNP_UNSUPPORTED, unsupported);
405 }
406}
407
408/*
409 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
410 *
411 * Returns < 0 if SEV is not supported, otherwise the position of the
412 * encryption bit in the page table descriptors.
413 */
414static int sev_check_cpu_support(void)
415{
416 unsigned int eax, ebx, ecx, edx;
417
418 /* Check for the SME/SEV support leaf */
419 eax = 0x80000000;
420 ecx = 0;
421 native_cpuid(&eax, &ebx, &ecx, &edx);
422 if (eax < 0x8000001f)
423 return -ENODEV;
424
425 /*
426 * Check for the SME/SEV feature:
427 * CPUID Fn8000_001F[EAX]
428 * - Bit 0 - Secure Memory Encryption support
429 * - Bit 1 - Secure Encrypted Virtualization support
430 * CPUID Fn8000_001F[EBX]
431 * - Bits 5:0 - Pagetable bit position used to indicate encryption
432 */
433 eax = 0x8000001f;
434 ecx = 0;
435 native_cpuid(&eax, &ebx, &ecx, &edx);
436 /* Check whether SEV is supported */
437 if (!(eax & BIT(1)))
438 return -ENODEV;
439
440 return ebx & 0x3f;
441}
442
443void sev_enable(struct boot_params *bp)
444{
445 struct msr m;
446 int bitpos;
447 bool snp;
448
449 /*
450 * bp->cc_blob_address should only be set by boot/compressed kernel.
451 * Initialize it to 0 to ensure that uninitialized values from
452 * buggy bootloaders aren't propagated.
453 */
454 if (bp)
455 bp->cc_blob_address = 0;
456
457 /*
458 * Do an initial SEV capability check before snp_init() which
459 * loads the CPUID page and the same checks afterwards are done
460 * without the hypervisor and are trustworthy.
461 *
462 * If the HV fakes SEV support, the guest will crash'n'burn
463 * which is good enough.
464 */
465
466 if (sev_check_cpu_support() < 0)
467 return;
468
469 /*
470 * Setup/preliminary detection of SNP. This will be sanity-checked
471 * against CPUID/MSR values later.
472 */
473 snp = snp_init(bp);
474
475 /* Now repeat the checks with the SNP CPUID table. */
476
477 bitpos = sev_check_cpu_support();
478 if (bitpos < 0) {
479 if (snp)
480 error("SEV-SNP support indicated by CC blob, but not CPUID.");
481 return;
482 }
483
484 /* Set the SME mask if this is an SEV guest. */
485 boot_rdmsr(MSR_AMD64_SEV, &m);
486 sev_status = m.q;
487 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
488 return;
489
490 /* Negotiate the GHCB protocol version. */
491 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
492 if (!sev_es_negotiate_protocol())
493 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
494 }
495
496 /*
497 * SNP is supported in v2 of the GHCB spec which mandates support for HV
498 * features.
499 */
500 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
501 if (!(get_hv_features() & GHCB_HV_FT_SNP))
502 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
503
504 enforce_vmpl0();
505 }
506
507 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
508 error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
509
510 sme_me_mask = BIT_ULL(bitpos);
511}
512
513/*
514 * sev_get_status - Retrieve the SEV status mask
515 *
516 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
517 * AMD64_SEV MSR.
518 */
519u64 sev_get_status(void)
520{
521 struct msr m;
522
523 if (sev_check_cpu_support() < 0)
524 return 0;
525
526 boot_rdmsr(MSR_AMD64_SEV, &m);
527 return m.q;
528}
529
530/* Search for Confidential Computing blob in the EFI config table. */
531static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
532{
533 unsigned long cfg_table_pa;
534 unsigned int cfg_table_len;
535 int ret;
536
537 ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len);
538 if (ret)
539 return NULL;
540
541 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa,
542 cfg_table_len,
543 EFI_CC_BLOB_GUID);
544}
545
546/*
547 * Initial set up of SNP relies on information provided by the
548 * Confidential Computing blob, which can be passed to the boot kernel
549 * by firmware/bootloader in the following ways:
550 *
551 * - via an entry in the EFI config table
552 * - via a setup_data structure, as defined by the Linux Boot Protocol
553 *
554 * Scan for the blob in that order.
555 */
556static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
557{
558 struct cc_blob_sev_info *cc_info;
559
560 cc_info = find_cc_blob_efi(bp);
561 if (cc_info)
562 goto found_cc_info;
563
564 cc_info = find_cc_blob_setup_data(bp);
565 if (!cc_info)
566 return NULL;
567
568found_cc_info:
569 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
570 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
571
572 return cc_info;
573}
574
575/*
576 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
577 * will verify the SNP CPUID/MSR bits.
578 */
579bool snp_init(struct boot_params *bp)
580{
581 struct cc_blob_sev_info *cc_info;
582
583 if (!bp)
584 return false;
585
586 cc_info = find_cc_blob(bp);
587 if (!cc_info)
588 return false;
589
590 /*
591 * If a SNP-specific Confidential Computing blob is present, then
592 * firmware/bootloader have indicated SNP support. Verifying this
593 * involves CPUID checks which will be more reliable if the SNP
594 * CPUID table is used. See comments over snp_setup_cpuid_table() for
595 * more details.
596 */
597 setup_cpuid_table(cc_info);
598
599 /*
600 * Pass run-time kernel a pointer to CC info via boot_params so EFI
601 * config table doesn't need to be searched again during early startup
602 * phase.
603 */
604 bp->cc_blob_address = (u32)(unsigned long)cc_info;
605
606 return true;
607}
608
609void sev_prep_identity_maps(unsigned long top_level_pgt)
610{
611 /*
612 * The Confidential Computing blob is used very early in uncompressed
613 * kernel to find the in-memory CPUID table to handle CPUID
614 * instructions. Make sure an identity-mapping exists so it can be
615 * accessed after switchover.
616 */
617 if (sev_snp_enabled()) {
618 unsigned long cc_info_pa = boot_params_ptr->cc_blob_address;
619 struct cc_blob_sev_info *cc_info;
620
621 kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info));
622
623 cc_info = (struct cc_blob_sev_info *)cc_info_pa;
624 kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len);
625 }
626
627 sev_verify_cbit(top_level_pgt);
628}