1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * AMD Encrypted Register State Support
  4 *
  5 * Author: Joerg Roedel <jroedel@suse.de>
  6 *
  7 * This file is not compiled stand-alone. It contains code shared
  8 * between the pre-decompression boot code and the running Linux kernel
  9 * and is included directly into both code-bases.
 10 */
 11
 12#ifndef __BOOT_COMPRESSED
 13#define error(v)	pr_err(v)
 14#define has_cpuflag(f)	boot_cpu_has(f)
 15#endif
 16
 17/* I/O parameters for CPUID-related helpers */
 18struct cpuid_leaf {
 19	u32 fn;
 20	u32 subfn;
 21	u32 eax;
 22	u32 ebx;
 23	u32 ecx;
 24	u32 edx;
 25};
 26
 27/*
 28 * Individual entries of the SNP CPUID table, as defined by the SNP
 29 * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
 30 */
 31struct snp_cpuid_fn {
 32	u32 eax_in;
 33	u32 ecx_in;
 34	u64 xcr0_in;
 35	u64 xss_in;
 36	u32 eax;
 37	u32 ebx;
 38	u32 ecx;
 39	u32 edx;
 40	u64 __reserved;
 41} __packed;
 42
 43/*
 44 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
 45 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
 46 * of 64 entries per CPUID table.
 47 */
 48#define SNP_CPUID_COUNT_MAX 64
 49
 50struct snp_cpuid_table {
 51	u32 count;
 52	u32 __reserved1;
 53	u64 __reserved2;
 54	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
 55} __packed;
 56
 57/*
 58 * Since feature negotiation related variables are set early in the boot
 59 * process they must reside in the .data section so as not to be zeroed
 60 * out when the .bss section is later cleared.
 61 *
 62 * GHCB protocol version negotiated with the hypervisor.
 63 */
 64static u16 ghcb_version __ro_after_init;
 65
 66/* Copy of the SNP firmware's CPUID page. */
 67static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
 68
 69/*
 70 * These will be initialized based on CPUID table so that non-present
 71 * all-zero leaves (for sparse tables) can be differentiated from
 72 * invalid/out-of-range leaves. This is needed since all-zero leaves
 73 * still need to be post-processed.
 74 */
 75static u32 cpuid_std_range_max __ro_after_init;
 76static u32 cpuid_hyp_range_max __ro_after_init;
 77static u32 cpuid_ext_range_max __ro_after_init;
 78
 79static bool __init sev_es_check_cpu_features(void)
 80{
 81	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
 82		error("RDRAND instruction not supported - no trusted source of randomness available\n");
 83		return false;
 84	}
 85
 86	return true;
 87}
 88
 89static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason)
 90{
 91	u64 val = GHCB_MSR_TERM_REQ;
 92
 93	/* Tell the hypervisor what went wrong. */
 94	val |= GHCB_SEV_TERM_REASON(set, reason);
 
 
 
 95
 96	/* Request Guest Termination from Hypvervisor */
 97	sev_es_wr_ghcb_msr(val);
 98	VMGEXIT();
 99
100	while (true)
101		asm volatile("hlt\n" : : : "memory");
102}
103
104/*
105 * The hypervisor features are available from GHCB version 2 onward.
106 */
107static u64 get_hv_features(void)
108{
109	u64 val;
110
111	if (ghcb_version < 2)
112		return 0;
113
114	sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
115	VMGEXIT();
116
117	val = sev_es_rd_ghcb_msr();
118	if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
119		return 0;
120
121	return GHCB_MSR_HV_FT_RESP_VAL(val);
122}
123
124static void snp_register_ghcb_early(unsigned long paddr)
125{
126	unsigned long pfn = paddr >> PAGE_SHIFT;
127	u64 val;
128
129	sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
130	VMGEXIT();
131
132	val = sev_es_rd_ghcb_msr();
133
134	/* If the response GPA is not ours then abort the guest */
135	if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
136	    (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
137		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
138}
139
140static bool sev_es_negotiate_protocol(void)
141{
142	u64 val;
143
144	/* Do the GHCB protocol version negotiation */
145	sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
146	VMGEXIT();
147	val = sev_es_rd_ghcb_msr();
148
149	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
150		return false;
151
152	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
153	    GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
154		return false;
155
156	ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
157
158	return true;
159}
160
161static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
162{
163	ghcb->save.sw_exit_code = 0;
164	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
165}
166
167static bool vc_decoding_needed(unsigned long exit_code)
168{
169	/* Exceptions don't require to decode the instruction */
170	return !(exit_code >= SVM_EXIT_EXCP_BASE &&
171		 exit_code <= SVM_EXIT_LAST_EXCP);
172}
173
174static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
175				      struct pt_regs *regs,
176				      unsigned long exit_code)
177{
178	enum es_result ret = ES_OK;
179
180	memset(ctxt, 0, sizeof(*ctxt));
181	ctxt->regs = regs;
182
183	if (vc_decoding_needed(exit_code))
184		ret = vc_decode_insn(ctxt);
185
186	return ret;
187}
188
189static void vc_finish_insn(struct es_em_ctxt *ctxt)
190{
191	ctxt->regs->ip += ctxt->insn.length;
192}
193
194static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
195{
196	u32 ret;
197
198	ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
199	if (!ret)
200		return ES_OK;
201
202	if (ret == 1) {
203		u64 info = ghcb->save.sw_exit_info_2;
204		unsigned long v = info & SVM_EVTINJ_VEC_MASK;
205
206		/* Check if exception information from hypervisor is sane. */
207		if ((info & SVM_EVTINJ_VALID) &&
208		    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
209		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
210			ctxt->fi.vector = v;
211
212			if (info & SVM_EVTINJ_VALID_ERR)
213				ctxt->fi.error_code = info >> 32;
214
215			return ES_EXCEPTION;
216		}
217	}
218
219	return ES_VMM_ERROR;
220}
221
222static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
223					  struct es_em_ctxt *ctxt,
224					  u64 exit_code, u64 exit_info_1,
225					  u64 exit_info_2)
226{
 
 
227	/* Fill in protocol and format specifiers */
228	ghcb->protocol_version = ghcb_version;
229	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
230
231	ghcb_set_sw_exit_code(ghcb, exit_code);
232	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
233	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
234
235	sev_es_wr_ghcb_msr(__pa(ghcb));
236	VMGEXIT();
237
238	return verify_exception_info(ghcb, ctxt);
239}
240
241static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
242{
243	u64 val;
244
245	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
246	VMGEXIT();
247	val = sev_es_rd_ghcb_msr();
248	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
249		return -EIO;
250
251	*reg = (val >> 32);
252
253	return 0;
254}
255
256static int sev_cpuid_hv(struct cpuid_leaf *leaf)
257{
258	int ret;
259
260	/*
261	 * MSR protocol does not support fetching non-zero subfunctions, but is
262	 * sufficient to handle current early-boot cases. Should that change,
263	 * make sure to report an error rather than ignoring the index and
264	 * grabbing random values. If this issue arises in the future, handling
265	 * can be added here to use GHCB-page protocol for cases that occur late
266	 * enough in boot that GHCB page is available.
267	 */
268	if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
269		return -EINVAL;
270
271	ret =         __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
272	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
273	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
274	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
275
276	return ret;
277}
278
279/*
280 * This may be called early while still running on the initial identity
281 * mapping. Use RIP-relative addressing to obtain the correct address
282 * while running with the initial identity mapping as well as the
283 * switch-over to kernel virtual addresses later.
284 */
285static const struct snp_cpuid_table *snp_cpuid_get_table(void)
286{
287	void *ptr;
288
289	asm ("lea cpuid_table_copy(%%rip), %0"
290	     : "=r" (ptr)
291	     : "p" (&cpuid_table_copy));
292
293	return ptr;
294}
295
296/*
297 * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
298 * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
299 * and 1 based on the corresponding features enabled by a particular
300 * combination of XCR0 and XSS registers so that a guest can look up the
301 * version corresponding to the features currently enabled in its XCR0/XSS
302 * registers. The only values that differ between these versions/table
303 * entries is the enabled XSAVE area size advertised via EBX.
304 *
305 * While hypervisors may choose to make use of this support, it is more
306 * robust/secure for a guest to simply find the entry corresponding to the
307 * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
308 * XSAVE area size using subfunctions 2 through 64, as documented in APM
309 * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
310 *
311 * Since base/legacy XSAVE area size is documented as 0x240, use that value
312 * directly rather than relying on the base size in the CPUID table.
313 *
314 * Return: XSAVE area size on success, 0 otherwise.
315 */
316static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
317{
318	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
319	u64 xfeatures_found = 0;
320	u32 xsave_size = 0x240;
321	int i;
322
323	for (i = 0; i < cpuid_table->count; i++) {
324		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
325
326		if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
327			continue;
328		if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
329			continue;
330		if (xfeatures_found & (BIT_ULL(e->ecx_in)))
331			continue;
332
333		xfeatures_found |= (BIT_ULL(e->ecx_in));
334
335		if (compacted)
336			xsave_size += e->eax;
337		else
338			xsave_size = max(xsave_size, e->eax + e->ebx);
339	}
340
341	/*
342	 * Either the guest set unsupported XCR0/XSS bits, or the corresponding
343	 * entries in the CPUID table were not present. This is not a valid
344	 * state to be in.
345	 */
346	if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
347		return 0;
348
349	return xsave_size;
350}
351
352static bool
353snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
354{
355	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
356	int i;
357
358	for (i = 0; i < cpuid_table->count; i++) {
359		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
360
361		if (e->eax_in != leaf->fn)
362			continue;
363
364		if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
365			continue;
366
367		/*
368		 * For 0xD subfunctions 0 and 1, only use the entry corresponding
369		 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
370		 * See the comments above snp_cpuid_calc_xsave_size() for more
371		 * details.
372		 */
373		if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
374			if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
375				continue;
376
377		leaf->eax = e->eax;
378		leaf->ebx = e->ebx;
379		leaf->ecx = e->ecx;
380		leaf->edx = e->edx;
381
382		return true;
383	}
384
385	return false;
386}
387
388static void snp_cpuid_hv(struct cpuid_leaf *leaf)
389{
390	if (sev_cpuid_hv(leaf))
391		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
392}
393
394static int snp_cpuid_postprocess(struct cpuid_leaf *leaf)
395{
396	struct cpuid_leaf leaf_hv = *leaf;
397
398	switch (leaf->fn) {
399	case 0x1:
400		snp_cpuid_hv(&leaf_hv);
401
402		/* initial APIC ID */
403		leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
404		/* APIC enabled bit */
405		leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
406
407		/* OSXSAVE enabled bit */
408		if (native_read_cr4() & X86_CR4_OSXSAVE)
409			leaf->ecx |= BIT(27);
410		break;
411	case 0x7:
412		/* OSPKE enabled bit */
413		leaf->ecx &= ~BIT(4);
414		if (native_read_cr4() & X86_CR4_PKE)
415			leaf->ecx |= BIT(4);
416		break;
417	case 0xB:
418		leaf_hv.subfn = 0;
419		snp_cpuid_hv(&leaf_hv);
420
421		/* extended APIC ID */
422		leaf->edx = leaf_hv.edx;
423		break;
424	case 0xD: {
425		bool compacted = false;
426		u64 xcr0 = 1, xss = 0;
427		u32 xsave_size;
428
429		if (leaf->subfn != 0 && leaf->subfn != 1)
430			return 0;
431
432		if (native_read_cr4() & X86_CR4_OSXSAVE)
433			xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
434		if (leaf->subfn == 1) {
435			/* Get XSS value if XSAVES is enabled. */
436			if (leaf->eax & BIT(3)) {
437				unsigned long lo, hi;
438
439				asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
440						     : "c" (MSR_IA32_XSS));
441				xss = (hi << 32) | lo;
442			}
443
444			/*
445			 * The PPR and APM aren't clear on what size should be
446			 * encoded in 0xD:0x1:EBX when compaction is not enabled
447			 * by either XSAVEC (feature bit 1) or XSAVES (feature
448			 * bit 3) since SNP-capable hardware has these feature
449			 * bits fixed as 1. KVM sets it to 0 in this case, but
450			 * to avoid this becoming an issue it's safer to simply
451			 * treat this as unsupported for SNP guests.
452			 */
453			if (!(leaf->eax & (BIT(1) | BIT(3))))
454				return -EINVAL;
455
456			compacted = true;
457		}
458
459		xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
460		if (!xsave_size)
461			return -EINVAL;
462
463		leaf->ebx = xsave_size;
 
 
 
 
 
 
 
 
 
464		}
465		break;
466	case 0x8000001E:
467		snp_cpuid_hv(&leaf_hv);
468
469		/* extended APIC ID */
470		leaf->eax = leaf_hv.eax;
471		/* compute ID */
472		leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
473		/* node ID */
474		leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
475		break;
476	default:
477		/* No fix-ups needed, use values as-is. */
478		break;
479	}
480
481	return 0;
482}
483
484/*
485 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
486 * should be treated as fatal by caller.
487 */
488static int snp_cpuid(struct cpuid_leaf *leaf)
489{
490	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
491
492	if (!cpuid_table->count)
493		return -EOPNOTSUPP;
494
495	if (!snp_cpuid_get_validated_func(leaf)) {
496		/*
497		 * Some hypervisors will avoid keeping track of CPUID entries
498		 * where all values are zero, since they can be handled the
499		 * same as out-of-range values (all-zero). This is useful here
500		 * as well as it allows virtually all guest configurations to
501		 * work using a single SNP CPUID table.
502		 *
503		 * To allow for this, there is a need to distinguish between
504		 * out-of-range entries and in-range zero entries, since the
505		 * CPUID table entries are only a template that may need to be
506		 * augmented with additional values for things like
507		 * CPU-specific information during post-processing. So if it's
508		 * not in the table, set the values to zero. Then, if they are
509		 * within a valid CPUID range, proceed with post-processing
510		 * using zeros as the initial values. Otherwise, skip
511		 * post-processing and just return zeros immediately.
512		 */
513		leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
514
515		/* Skip post-processing for out-of-range zero leafs. */
516		if (!(leaf->fn <= cpuid_std_range_max ||
517		      (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) ||
518		      (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))
519			return 0;
520	}
521
522	return snp_cpuid_postprocess(leaf);
523}
524
525/*
526 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
527 * page yet, so it only supports the MSR based communication with the
528 * hypervisor and only the CPUID exit-code.
529 */
530void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
531{
532	unsigned int subfn = lower_bits(regs->cx, 32);
533	unsigned int fn = lower_bits(regs->ax, 32);
534	struct cpuid_leaf leaf;
535	int ret;
536
537	/* Only CPUID is supported via MSR protocol */
538	if (exit_code != SVM_EXIT_CPUID)
539		goto fail;
540
541	leaf.fn = fn;
542	leaf.subfn = subfn;
543
544	ret = snp_cpuid(&leaf);
545	if (!ret)
546		goto cpuid_done;
547
548	if (ret != -EOPNOTSUPP)
 
 
 
549		goto fail;
 
550
551	if (sev_cpuid_hv(&leaf))
 
 
 
552		goto fail;
 
553
554cpuid_done:
555	regs->ax = leaf.eax;
556	regs->bx = leaf.ebx;
557	regs->cx = leaf.ecx;
558	regs->dx = leaf.edx;
 
559
560	/*
561	 * This is a VC handler and the #VC is only raised when SEV-ES is
562	 * active, which means SEV must be active too. Do sanity checks on the
563	 * CPUID results to make sure the hypervisor does not trick the kernel
564	 * into the no-sev path. This could map sensitive data unencrypted and
565	 * make it accessible to the hypervisor.
566	 *
567	 * In particular, check for:
568	 *	- Availability of CPUID leaf 0x8000001f
569	 *	- SEV CPUID bit.
570	 *
571	 * The hypervisor might still report the wrong C-bit position, but this
572	 * can't be checked here.
573	 */
574
575	if (fn == 0x80000000 && (regs->ax < 0x8000001f))
576		/* SEV leaf check */
577		goto fail;
578	else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
579		/* SEV bit */
580		goto fail;
581
582	/* Skip over the CPUID two-byte opcode */
583	regs->ip += 2;
584
585	return;
586
587fail:
588	/* Terminate the guest */
589	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
590}
591
592static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
593					  void *src, char *buf,
594					  unsigned int data_size,
595					  unsigned int count,
596					  bool backwards)
597{
598	int i, b = backwards ? -1 : 1;
599	enum es_result ret = ES_OK;
600
601	for (i = 0; i < count; i++) {
602		void *s = src + (i * data_size * b);
603		char *d = buf + (i * data_size);
604
605		ret = vc_read_mem(ctxt, s, d, data_size);
606		if (ret != ES_OK)
607			break;
608	}
609
610	return ret;
611}
612
613static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
614					   void *dst, char *buf,
615					   unsigned int data_size,
616					   unsigned int count,
617					   bool backwards)
618{
619	int i, s = backwards ? -1 : 1;
620	enum es_result ret = ES_OK;
621
622	for (i = 0; i < count; i++) {
623		void *d = dst + (i * data_size * s);
624		char *b = buf + (i * data_size);
625
626		ret = vc_write_mem(ctxt, d, b, data_size);
627		if (ret != ES_OK)
628			break;
629	}
630
631	return ret;
632}
633
634#define IOIO_TYPE_STR  BIT(2)
635#define IOIO_TYPE_IN   1
636#define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
637#define IOIO_TYPE_OUT  0
638#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
639
640#define IOIO_REP       BIT(3)
641
642#define IOIO_ADDR_64   BIT(9)
643#define IOIO_ADDR_32   BIT(8)
644#define IOIO_ADDR_16   BIT(7)
645
646#define IOIO_DATA_32   BIT(6)
647#define IOIO_DATA_16   BIT(5)
648#define IOIO_DATA_8    BIT(4)
649
650#define IOIO_SEG_ES    (0 << 10)
651#define IOIO_SEG_DS    (3 << 10)
652
653static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
654{
655	struct insn *insn = &ctxt->insn;
656	*exitinfo = 0;
657
658	switch (insn->opcode.bytes[0]) {
659	/* INS opcodes */
660	case 0x6c:
661	case 0x6d:
662		*exitinfo |= IOIO_TYPE_INS;
663		*exitinfo |= IOIO_SEG_ES;
664		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
665		break;
666
667	/* OUTS opcodes */
668	case 0x6e:
669	case 0x6f:
670		*exitinfo |= IOIO_TYPE_OUTS;
671		*exitinfo |= IOIO_SEG_DS;
672		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
673		break;
674
675	/* IN immediate opcodes */
676	case 0xe4:
677	case 0xe5:
678		*exitinfo |= IOIO_TYPE_IN;
679		*exitinfo |= (u8)insn->immediate.value << 16;
680		break;
681
682	/* OUT immediate opcodes */
683	case 0xe6:
684	case 0xe7:
685		*exitinfo |= IOIO_TYPE_OUT;
686		*exitinfo |= (u8)insn->immediate.value << 16;
687		break;
688
689	/* IN register opcodes */
690	case 0xec:
691	case 0xed:
692		*exitinfo |= IOIO_TYPE_IN;
693		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
694		break;
695
696	/* OUT register opcodes */
697	case 0xee:
698	case 0xef:
699		*exitinfo |= IOIO_TYPE_OUT;
700		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
701		break;
702
703	default:
704		return ES_DECODE_FAILED;
705	}
706
707	switch (insn->opcode.bytes[0]) {
708	case 0x6c:
709	case 0x6e:
710	case 0xe4:
711	case 0xe6:
712	case 0xec:
713	case 0xee:
714		/* Single byte opcodes */
715		*exitinfo |= IOIO_DATA_8;
716		break;
717	default:
718		/* Length determined by instruction parsing */
719		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
720						     : IOIO_DATA_32;
721	}
722	switch (insn->addr_bytes) {
723	case 2:
724		*exitinfo |= IOIO_ADDR_16;
725		break;
726	case 4:
727		*exitinfo |= IOIO_ADDR_32;
728		break;
729	case 8:
730		*exitinfo |= IOIO_ADDR_64;
731		break;
732	}
733
734	if (insn_has_rep_prefix(insn))
735		*exitinfo |= IOIO_REP;
736
737	return ES_OK;
738}
739
740static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
741{
742	struct pt_regs *regs = ctxt->regs;
743	u64 exit_info_1, exit_info_2;
744	enum es_result ret;
745
746	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
747	if (ret != ES_OK)
748		return ret;
749
750	if (exit_info_1 & IOIO_TYPE_STR) {
751
752		/* (REP) INS/OUTS */
753
754		bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
755		unsigned int io_bytes, exit_bytes;
756		unsigned int ghcb_count, op_count;
757		unsigned long es_base;
758		u64 sw_scratch;
759
760		/*
761		 * For the string variants with rep prefix the amount of in/out
762		 * operations per #VC exception is limited so that the kernel
763		 * has a chance to take interrupts and re-schedule while the
764		 * instruction is emulated.
765		 */
766		io_bytes   = (exit_info_1 >> 4) & 0x7;
767		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
768
769		op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
770		exit_info_2 = min(op_count, ghcb_count);
771		exit_bytes  = exit_info_2 * io_bytes;
772
773		es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
774
775		/* Read bytes of OUTS into the shared buffer */
776		if (!(exit_info_1 & IOIO_TYPE_IN)) {
777			ret = vc_insn_string_read(ctxt,
778					       (void *)(es_base + regs->si),
779					       ghcb->shared_buffer, io_bytes,
780					       exit_info_2, df);
781			if (ret)
782				return ret;
783		}
784
785		/*
786		 * Issue an VMGEXIT to the HV to consume the bytes from the
787		 * shared buffer or to have it write them into the shared buffer
788		 * depending on the instruction: OUTS or INS.
789		 */
790		sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
791		ghcb_set_sw_scratch(ghcb, sw_scratch);
792		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
793					  exit_info_1, exit_info_2);
794		if (ret != ES_OK)
795			return ret;
796
797		/* Read bytes from shared buffer into the guest's destination. */
798		if (exit_info_1 & IOIO_TYPE_IN) {
799			ret = vc_insn_string_write(ctxt,
800						   (void *)(es_base + regs->di),
801						   ghcb->shared_buffer, io_bytes,
802						   exit_info_2, df);
803			if (ret)
804				return ret;
805
806			if (df)
807				regs->di -= exit_bytes;
808			else
809				regs->di += exit_bytes;
810		} else {
811			if (df)
812				regs->si -= exit_bytes;
813			else
814				regs->si += exit_bytes;
815		}
816
817		if (exit_info_1 & IOIO_REP)
818			regs->cx -= exit_info_2;
819
820		ret = regs->cx ? ES_RETRY : ES_OK;
821
822	} else {
823
824		/* IN/OUT into/from rAX */
825
826		int bits = (exit_info_1 & 0x70) >> 1;
827		u64 rax = 0;
828
829		if (!(exit_info_1 & IOIO_TYPE_IN))
830			rax = lower_bits(regs->ax, bits);
831
832		ghcb_set_rax(ghcb, rax);
833
834		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
835		if (ret != ES_OK)
836			return ret;
837
838		if (exit_info_1 & IOIO_TYPE_IN) {
839			if (!ghcb_rax_is_valid(ghcb))
840				return ES_VMM_ERROR;
841			regs->ax = lower_bits(ghcb->save.rax, bits);
842		}
843	}
844
845	return ret;
846}
847
848static int vc_handle_cpuid_snp(struct pt_regs *regs)
849{
850	struct cpuid_leaf leaf;
851	int ret;
852
853	leaf.fn = regs->ax;
854	leaf.subfn = regs->cx;
855	ret = snp_cpuid(&leaf);
856	if (!ret) {
857		regs->ax = leaf.eax;
858		regs->bx = leaf.ebx;
859		regs->cx = leaf.ecx;
860		regs->dx = leaf.edx;
861	}
862
863	return ret;
864}
865
866static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
867				      struct es_em_ctxt *ctxt)
868{
869	struct pt_regs *regs = ctxt->regs;
870	u32 cr4 = native_read_cr4();
871	enum es_result ret;
872	int snp_cpuid_ret;
873
874	snp_cpuid_ret = vc_handle_cpuid_snp(regs);
875	if (!snp_cpuid_ret)
876		return ES_OK;
877	if (snp_cpuid_ret != -EOPNOTSUPP)
878		return ES_VMM_ERROR;
879
880	ghcb_set_rax(ghcb, regs->ax);
881	ghcb_set_rcx(ghcb, regs->cx);
882
883	if (cr4 & X86_CR4_OSXSAVE)
884		/* Safe to read xcr0 */
885		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
886	else
887		/* xgetbv will cause #GP - use reset value for xcr0 */
888		ghcb_set_xcr0(ghcb, 1);
889
890	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
891	if (ret != ES_OK)
892		return ret;
893
894	if (!(ghcb_rax_is_valid(ghcb) &&
895	      ghcb_rbx_is_valid(ghcb) &&
896	      ghcb_rcx_is_valid(ghcb) &&
897	      ghcb_rdx_is_valid(ghcb)))
898		return ES_VMM_ERROR;
899
900	regs->ax = ghcb->save.rax;
901	regs->bx = ghcb->save.rbx;
902	regs->cx = ghcb->save.rcx;
903	regs->dx = ghcb->save.rdx;
904
905	return ES_OK;
906}
907
908static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
909				      struct es_em_ctxt *ctxt,
910				      unsigned long exit_code)
911{
912	bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
913	enum es_result ret;
914
915	ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
916	if (ret != ES_OK)
917		return ret;
918
919	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
920	     (!rdtscp || ghcb_rcx_is_valid(ghcb))))
921		return ES_VMM_ERROR;
922
923	ctxt->regs->ax = ghcb->save.rax;
924	ctxt->regs->dx = ghcb->save.rdx;
925	if (rdtscp)
926		ctxt->regs->cx = ghcb->save.rcx;
927
928	return ES_OK;
929}
930
931struct cc_setup_data {
932	struct setup_data header;
933	u32 cc_blob_address;
934};
935
936/*
937 * Search for a Confidential Computing blob passed in as a setup_data entry
938 * via the Linux Boot Protocol.
939 */
940static struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
941{
942	struct cc_setup_data *sd = NULL;
943	struct setup_data *hdr;
944
945	hdr = (struct setup_data *)bp->hdr.setup_data;
946
947	while (hdr) {
948		if (hdr->type == SETUP_CC_BLOB) {
949			sd = (struct cc_setup_data *)hdr;
950			return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
951		}
952		hdr = (struct setup_data *)hdr->next;
953	}
954
955	return NULL;
956}
957
958/*
959 * Initialize the kernel's copy of the SNP CPUID table, and set up the
960 * pointer that will be used to access it.
961 *
962 * Maintaining a direct mapping of the SNP CPUID table used by firmware would
963 * be possible as an alternative, but the approach is brittle since the
964 * mapping needs to be updated in sync with all the changes to virtual memory
965 * layout and related mapping facilities throughout the boot process.
966 */
967static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
968{
969	const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
970	int i;
971
972	if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
973		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
974
975	cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
976	if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
977		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
978
979	cpuid_table = snp_cpuid_get_table();
980	memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
981
982	/* Initialize CPUID ranges for range-checking. */
983	for (i = 0; i < cpuid_table->count; i++) {
984		const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
985
986		if (fn->eax_in == 0x0)
987			cpuid_std_range_max = fn->eax;
988		else if (fn->eax_in == 0x40000000)
989			cpuid_hyp_range_max = fn->eax;
990		else if (fn->eax_in == 0x80000000)
991			cpuid_ext_range_max = fn->eax;
992	}
993}