Linux Audio

Check our new training course

Loading...
Note: File does not exist in v3.1.
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * AMD Encrypted Register State Support
  4 *
  5 * Author: Joerg Roedel <jroedel@suse.de>
  6 */
  7
  8/*
  9 * misc.h needs to be first because it knows how to include the other kernel
 10 * headers in the pre-decompression code in a way that does not break
 11 * compilation.
 12 */
 13#include "misc.h"
 14
 15#include <asm/pgtable_types.h>
 16#include <asm/sev.h>
 17#include <asm/trapnr.h>
 18#include <asm/trap_pf.h>
 19#include <asm/msr-index.h>
 20#include <asm/fpu/xcr.h>
 21#include <asm/ptrace.h>
 22#include <asm/svm.h>
 23#include <asm/cpuid.h>
 24
 25#include "error.h"
 26#include "../msr.h"
 27
 28static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
 29struct ghcb *boot_ghcb;
 30
 31/*
 32 * Copy a version of this function here - insn-eval.c can't be used in
 33 * pre-decompression code.
 34 */
 35static bool insn_has_rep_prefix(struct insn *insn)
 36{
 37	insn_byte_t p;
 38	int i;
 39
 40	insn_get_prefixes(insn);
 41
 42	for_each_insn_prefix(insn, i, p) {
 43		if (p == 0xf2 || p == 0xf3)
 44			return true;
 45	}
 46
 47	return false;
 48}
 49
 50/*
 51 * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
 52 * doesn't use segments.
 53 */
 54static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
 55{
 56	return 0UL;
 57}
 58
 59static inline u64 sev_es_rd_ghcb_msr(void)
 60{
 61	struct msr m;
 62
 63	boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
 64
 65	return m.q;
 66}
 67
 68static inline void sev_es_wr_ghcb_msr(u64 val)
 69{
 70	struct msr m;
 71
 72	m.q = val;
 73	boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
 74}
 75
 76static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
 77{
 78	char buffer[MAX_INSN_SIZE];
 79	int ret;
 80
 81	memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
 82
 83	ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
 84	if (ret < 0)
 85		return ES_DECODE_FAILED;
 86
 87	return ES_OK;
 88}
 89
 90static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
 91				   void *dst, char *buf, size_t size)
 92{
 93	memcpy(dst, buf, size);
 94
 95	return ES_OK;
 96}
 97
 98static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
 99				  void *src, char *buf, size_t size)
100{
101	memcpy(buf, src, size);
102
103	return ES_OK;
104}
105
106static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
107{
108	return ES_OK;
109}
110
111static bool fault_in_kernel_space(unsigned long address)
112{
113	return false;
114}
115
116#undef __init
117#define __init
118
119#define __BOOT_COMPRESSED
120
121/* Basic instruction decoding support needed */
122#include "../../lib/inat.c"
123#include "../../lib/insn.c"
124
125/* Include code for early handlers */
126#include "../../kernel/sev-shared.c"
127
128bool sev_snp_enabled(void)
129{
130	return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
131}
132
133static void __page_state_change(unsigned long paddr, enum psc_op op)
134{
135	u64 val;
136
137	if (!sev_snp_enabled())
138		return;
139
140	/*
141	 * If private -> shared then invalidate the page before requesting the
142	 * state change in the RMP table.
143	 */
144	if (op == SNP_PAGE_STATE_SHARED && pvalidate(paddr, RMP_PG_SIZE_4K, 0))
145		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
146
147	/* Issue VMGEXIT to change the page state in RMP table. */
148	sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
149	VMGEXIT();
150
151	/* Read the response of the VMGEXIT. */
152	val = sev_es_rd_ghcb_msr();
153	if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
154		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
155
156	/*
157	 * Now that page state is changed in the RMP table, validate it so that it is
158	 * consistent with the RMP entry.
159	 */
160	if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(paddr, RMP_PG_SIZE_4K, 1))
161		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
162}
163
164void snp_set_page_private(unsigned long paddr)
165{
166	__page_state_change(paddr, SNP_PAGE_STATE_PRIVATE);
167}
168
169void snp_set_page_shared(unsigned long paddr)
170{
171	__page_state_change(paddr, SNP_PAGE_STATE_SHARED);
172}
173
174static bool early_setup_ghcb(void)
175{
176	if (set_page_decrypted((unsigned long)&boot_ghcb_page))
177		return false;
178
179	/* Page is now mapped decrypted, clear it */
180	memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page));
181
182	boot_ghcb = &boot_ghcb_page;
183
184	/* Initialize lookup tables for the instruction decoder */
185	inat_init_tables();
186
187	/* SNP guest requires the GHCB GPA must be registered */
188	if (sev_snp_enabled())
189		snp_register_ghcb_early(__pa(&boot_ghcb_page));
190
191	return true;
192}
193
194static phys_addr_t __snp_accept_memory(struct snp_psc_desc *desc,
195				       phys_addr_t pa, phys_addr_t pa_end)
196{
197	struct psc_hdr *hdr;
198	struct psc_entry *e;
199	unsigned int i;
200
201	hdr = &desc->hdr;
202	memset(hdr, 0, sizeof(*hdr));
203
204	e = desc->entries;
205
206	i = 0;
207	while (pa < pa_end && i < VMGEXIT_PSC_MAX_ENTRY) {
208		hdr->end_entry = i;
209
210		e->gfn = pa >> PAGE_SHIFT;
211		e->operation = SNP_PAGE_STATE_PRIVATE;
212		if (IS_ALIGNED(pa, PMD_SIZE) && (pa_end - pa) >= PMD_SIZE) {
213			e->pagesize = RMP_PG_SIZE_2M;
214			pa += PMD_SIZE;
215		} else {
216			e->pagesize = RMP_PG_SIZE_4K;
217			pa += PAGE_SIZE;
218		}
219
220		e++;
221		i++;
222	}
223
224	if (vmgexit_psc(boot_ghcb, desc))
225		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
226
227	pvalidate_pages(desc);
228
229	return pa;
230}
231
232void snp_accept_memory(phys_addr_t start, phys_addr_t end)
233{
234	struct snp_psc_desc desc = {};
235	unsigned int i;
236	phys_addr_t pa;
237
238	if (!boot_ghcb && !early_setup_ghcb())
239		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
240
241	pa = start;
242	while (pa < end)
243		pa = __snp_accept_memory(&desc, pa, end);
244}
245
246void sev_es_shutdown_ghcb(void)
247{
248	if (!boot_ghcb)
249		return;
250
251	if (!sev_es_check_cpu_features())
252		error("SEV-ES CPU Features missing.");
253
254	/*
255	 * GHCB Page must be flushed from the cache and mapped encrypted again.
256	 * Otherwise the running kernel will see strange cache effects when
257	 * trying to use that page.
258	 */
259	if (set_page_encrypted((unsigned long)&boot_ghcb_page))
260		error("Can't map GHCB page encrypted");
261
262	/*
263	 * GHCB page is mapped encrypted again and flushed from the cache.
264	 * Mark it non-present now to catch bugs when #VC exceptions trigger
265	 * after this point.
266	 */
267	if (set_page_non_present((unsigned long)&boot_ghcb_page))
268		error("Can't unmap GHCB page");
269}
270
271static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set,
272					     unsigned int reason, u64 exit_info_2)
273{
274	u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason);
275
276	vc_ghcb_invalidate(ghcb);
277	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST);
278	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
279	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
280
281	sev_es_wr_ghcb_msr(__pa(ghcb));
282	VMGEXIT();
283
284	while (true)
285		asm volatile("hlt\n" : : : "memory");
286}
287
288bool sev_es_check_ghcb_fault(unsigned long address)
289{
290	/* Check whether the fault was on the GHCB page */
291	return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
292}
293
294void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
295{
296	struct es_em_ctxt ctxt;
297	enum es_result result;
298
299	if (!boot_ghcb && !early_setup_ghcb())
300		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
301
302	vc_ghcb_invalidate(boot_ghcb);
303	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
304	if (result != ES_OK)
305		goto finish;
306
307	switch (exit_code) {
308	case SVM_EXIT_RDTSC:
309	case SVM_EXIT_RDTSCP:
310		result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
311		break;
312	case SVM_EXIT_IOIO:
313		result = vc_handle_ioio(boot_ghcb, &ctxt);
314		break;
315	case SVM_EXIT_CPUID:
316		result = vc_handle_cpuid(boot_ghcb, &ctxt);
317		break;
318	default:
319		result = ES_UNSUPPORTED;
320		break;
321	}
322
323finish:
324	if (result == ES_OK)
325		vc_finish_insn(&ctxt);
326	else if (result != ES_RETRY)
327		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
328}
329
330static void enforce_vmpl0(void)
331{
332	u64 attrs;
333	int err;
334
335	/*
336	 * RMPADJUST modifies RMP permissions of a lesser-privileged (numerically
337	 * higher) privilege level. Here, clear the VMPL1 permission mask of the
338	 * GHCB page. If the guest is not running at VMPL0, this will fail.
339	 *
340	 * If the guest is running at VMPL0, it will succeed. Even if that operation
341	 * modifies permission bits, it is still ok to do so currently because Linux
342	 * SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks
343	 * changing is a don't-care.
344	 */
345	attrs = 1;
346	if (rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs))
347		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
348}
349
350/*
351 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
352 * guest side implementation for proper functioning of the guest. If any
353 * of these features are enabled in the hypervisor but are lacking guest
354 * side implementation, the behavior of the guest will be undefined. The
355 * guest could fail in non-obvious way making it difficult to debug.
356 *
357 * As the behavior of reserved feature bits is unknown to be on the
358 * safe side add them to the required features mask.
359 */
360#define SNP_FEATURES_IMPL_REQ	(MSR_AMD64_SNP_VTOM |			\
361				 MSR_AMD64_SNP_REFLECT_VC |		\
362				 MSR_AMD64_SNP_RESTRICTED_INJ |		\
363				 MSR_AMD64_SNP_ALT_INJ |		\
364				 MSR_AMD64_SNP_DEBUG_SWAP |		\
365				 MSR_AMD64_SNP_VMPL_SSS |		\
366				 MSR_AMD64_SNP_SECURE_TSC |		\
367				 MSR_AMD64_SNP_VMGEXIT_PARAM |		\
368				 MSR_AMD64_SNP_VMSA_REG_PROTECTION |	\
369				 MSR_AMD64_SNP_RESERVED_BIT13 |		\
370				 MSR_AMD64_SNP_RESERVED_BIT15 |		\
371				 MSR_AMD64_SNP_RESERVED_MASK)
372
373/*
374 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
375 * by the guest kernel. As and when a new feature is implemented in the
376 * guest kernel, a corresponding bit should be added to the mask.
377 */
378#define SNP_FEATURES_PRESENT	MSR_AMD64_SNP_DEBUG_SWAP
379
380u64 snp_get_unsupported_features(u64 status)
381{
382	if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
383		return 0;
384
385	return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
386}
387
388void snp_check_features(void)
389{
390	u64 unsupported;
391
392	/*
393	 * Terminate the boot if hypervisor has enabled any feature lacking
394	 * guest side implementation. Pass on the unsupported features mask through
395	 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
396	 * as part of the guest boot failure.
397	 */
398	unsupported = snp_get_unsupported_features(sev_status);
399	if (unsupported) {
400		if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb()))
401			sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
402
403		sev_es_ghcb_terminate(boot_ghcb, SEV_TERM_SET_GEN,
404				      GHCB_SNP_UNSUPPORTED, unsupported);
405	}
406}
407
408/*
409 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
410 *
411 * Returns < 0 if SEV is not supported, otherwise the position of the
412 * encryption bit in the page table descriptors.
413 */
414static int sev_check_cpu_support(void)
415{
416	unsigned int eax, ebx, ecx, edx;
417
418	/* Check for the SME/SEV support leaf */
419	eax = 0x80000000;
420	ecx = 0;
421	native_cpuid(&eax, &ebx, &ecx, &edx);
422	if (eax < 0x8000001f)
423		return -ENODEV;
424
425	/*
426	 * Check for the SME/SEV feature:
427	 *   CPUID Fn8000_001F[EAX]
428	 *   - Bit 0 - Secure Memory Encryption support
429	 *   - Bit 1 - Secure Encrypted Virtualization support
430	 *   CPUID Fn8000_001F[EBX]
431	 *   - Bits 5:0 - Pagetable bit position used to indicate encryption
432	 */
433	eax = 0x8000001f;
434	ecx = 0;
435	native_cpuid(&eax, &ebx, &ecx, &edx);
436	/* Check whether SEV is supported */
437	if (!(eax & BIT(1)))
438		return -ENODEV;
439
440	return ebx & 0x3f;
441}
442
443void sev_enable(struct boot_params *bp)
444{
445	struct msr m;
446	int bitpos;
447	bool snp;
448
449	/*
450	 * bp->cc_blob_address should only be set by boot/compressed kernel.
451	 * Initialize it to 0 to ensure that uninitialized values from
452	 * buggy bootloaders aren't propagated.
453	 */
454	if (bp)
455		bp->cc_blob_address = 0;
456
457	/*
458	 * Do an initial SEV capability check before snp_init() which
459	 * loads the CPUID page and the same checks afterwards are done
460	 * without the hypervisor and are trustworthy.
461	 *
462	 * If the HV fakes SEV support, the guest will crash'n'burn
463	 * which is good enough.
464	 */
465
466	if (sev_check_cpu_support() < 0)
467		return;
468
469	/*
470	 * Setup/preliminary detection of SNP. This will be sanity-checked
471	 * against CPUID/MSR values later.
472	 */
473	snp = snp_init(bp);
474
475	/* Now repeat the checks with the SNP CPUID table. */
476
477	bitpos = sev_check_cpu_support();
478	if (bitpos < 0) {
479		if (snp)
480			error("SEV-SNP support indicated by CC blob, but not CPUID.");
481		return;
482	}
483
484	/* Set the SME mask if this is an SEV guest. */
485	boot_rdmsr(MSR_AMD64_SEV, &m);
486	sev_status = m.q;
487	if (!(sev_status & MSR_AMD64_SEV_ENABLED))
488		return;
489
490	/* Negotiate the GHCB protocol version. */
491	if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
492		if (!sev_es_negotiate_protocol())
493			sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
494	}
495
496	/*
497	 * SNP is supported in v2 of the GHCB spec which mandates support for HV
498	 * features.
499	 */
500	if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
501		if (!(get_hv_features() & GHCB_HV_FT_SNP))
502			sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
503
504		enforce_vmpl0();
505	}
506
507	if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
508		error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
509
510	sme_me_mask = BIT_ULL(bitpos);
511}
512
513/*
514 * sev_get_status - Retrieve the SEV status mask
515 *
516 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
517 * AMD64_SEV MSR.
518 */
519u64 sev_get_status(void)
520{
521	struct msr m;
522
523	if (sev_check_cpu_support() < 0)
524		return 0;
525
526	boot_rdmsr(MSR_AMD64_SEV, &m);
527	return m.q;
528}
529
530/* Search for Confidential Computing blob in the EFI config table. */
531static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
532{
533	unsigned long cfg_table_pa;
534	unsigned int cfg_table_len;
535	int ret;
536
537	ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len);
538	if (ret)
539		return NULL;
540
541	return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa,
542								cfg_table_len,
543								EFI_CC_BLOB_GUID);
544}
545
546/*
547 * Initial set up of SNP relies on information provided by the
548 * Confidential Computing blob, which can be passed to the boot kernel
549 * by firmware/bootloader in the following ways:
550 *
551 * - via an entry in the EFI config table
552 * - via a setup_data structure, as defined by the Linux Boot Protocol
553 *
554 * Scan for the blob in that order.
555 */
556static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
557{
558	struct cc_blob_sev_info *cc_info;
559
560	cc_info = find_cc_blob_efi(bp);
561	if (cc_info)
562		goto found_cc_info;
563
564	cc_info = find_cc_blob_setup_data(bp);
565	if (!cc_info)
566		return NULL;
567
568found_cc_info:
569	if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
570		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
571
572	return cc_info;
573}
574
575/*
576 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
577 * will verify the SNP CPUID/MSR bits.
578 */
579bool snp_init(struct boot_params *bp)
580{
581	struct cc_blob_sev_info *cc_info;
582
583	if (!bp)
584		return false;
585
586	cc_info = find_cc_blob(bp);
587	if (!cc_info)
588		return false;
589
590	/*
591	 * If a SNP-specific Confidential Computing blob is present, then
592	 * firmware/bootloader have indicated SNP support. Verifying this
593	 * involves CPUID checks which will be more reliable if the SNP
594	 * CPUID table is used. See comments over snp_setup_cpuid_table() for
595	 * more details.
596	 */
597	setup_cpuid_table(cc_info);
598
599	/*
600	 * Pass run-time kernel a pointer to CC info via boot_params so EFI
601	 * config table doesn't need to be searched again during early startup
602	 * phase.
603	 */
604	bp->cc_blob_address = (u32)(unsigned long)cc_info;
605
606	return true;
607}
608
609void sev_prep_identity_maps(unsigned long top_level_pgt)
610{
611	/*
612	 * The Confidential Computing blob is used very early in uncompressed
613	 * kernel to find the in-memory CPUID table to handle CPUID
614	 * instructions. Make sure an identity-mapping exists so it can be
615	 * accessed after switchover.
616	 */
617	if (sev_snp_enabled()) {
618		unsigned long cc_info_pa = boot_params_ptr->cc_blob_address;
619		struct cc_blob_sev_info *cc_info;
620
621		kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info));
622
623		cc_info = (struct cc_blob_sev_info *)cc_info_pa;
624		kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len);
625	}
626
627	sev_verify_cbit(top_level_pgt);
628}