Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 *
7 * This file is not compiled stand-alone. It contains code shared
8 * between the pre-decompression boot code and the running Linux kernel
9 * and is included directly into both code-bases.
10 */
11
12#ifndef __BOOT_COMPRESSED
13#define error(v) pr_err(v)
14#define has_cpuflag(f) boot_cpu_has(f)
15#endif
16
17static bool __init sev_es_check_cpu_features(void)
18{
19 if (!has_cpuflag(X86_FEATURE_RDRAND)) {
20 error("RDRAND instruction not supported - no trusted source of randomness available\n");
21 return false;
22 }
23
24 return true;
25}
26
27static void __noreturn sev_es_terminate(unsigned int reason)
28{
29 u64 val = GHCB_MSR_TERM_REQ;
30
31 /*
32 * Tell the hypervisor what went wrong - only reason-set 0 is
33 * currently supported.
34 */
35 val |= GHCB_SEV_TERM_REASON(0, reason);
36
37 /* Request Guest Termination from Hypvervisor */
38 sev_es_wr_ghcb_msr(val);
39 VMGEXIT();
40
41 while (true)
42 asm volatile("hlt\n" : : : "memory");
43}
44
45static bool sev_es_negotiate_protocol(void)
46{
47 u64 val;
48
49 /* Do the GHCB protocol version negotiation */
50 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
51 VMGEXIT();
52 val = sev_es_rd_ghcb_msr();
53
54 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
55 return false;
56
57 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTO_OUR ||
58 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTO_OUR)
59 return false;
60
61 return true;
62}
63
64static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
65{
66 ghcb->save.sw_exit_code = 0;
67 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
68}
69
70static bool vc_decoding_needed(unsigned long exit_code)
71{
72 /* Exceptions don't require to decode the instruction */
73 return !(exit_code >= SVM_EXIT_EXCP_BASE &&
74 exit_code <= SVM_EXIT_LAST_EXCP);
75}
76
77static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
78 struct pt_regs *regs,
79 unsigned long exit_code)
80{
81 enum es_result ret = ES_OK;
82
83 memset(ctxt, 0, sizeof(*ctxt));
84 ctxt->regs = regs;
85
86 if (vc_decoding_needed(exit_code))
87 ret = vc_decode_insn(ctxt);
88
89 return ret;
90}
91
92static void vc_finish_insn(struct es_em_ctxt *ctxt)
93{
94 ctxt->regs->ip += ctxt->insn.length;
95}
96
97static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
98 struct es_em_ctxt *ctxt,
99 u64 exit_code, u64 exit_info_1,
100 u64 exit_info_2)
101{
102 enum es_result ret;
103
104 /* Fill in protocol and format specifiers */
105 ghcb->protocol_version = GHCB_PROTOCOL_MAX;
106 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
107
108 ghcb_set_sw_exit_code(ghcb, exit_code);
109 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
110 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
111
112 sev_es_wr_ghcb_msr(__pa(ghcb));
113 VMGEXIT();
114
115 if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
116 u64 info = ghcb->save.sw_exit_info_2;
117 unsigned long v;
118
119 info = ghcb->save.sw_exit_info_2;
120 v = info & SVM_EVTINJ_VEC_MASK;
121
122 /* Check if exception information from hypervisor is sane. */
123 if ((info & SVM_EVTINJ_VALID) &&
124 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
125 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
126 ctxt->fi.vector = v;
127 if (info & SVM_EVTINJ_VALID_ERR)
128 ctxt->fi.error_code = info >> 32;
129 ret = ES_EXCEPTION;
130 } else {
131 ret = ES_VMM_ERROR;
132 }
133 } else if (ghcb->save.sw_exit_info_1 & 0xffffffff) {
134 ret = ES_VMM_ERROR;
135 } else {
136 ret = ES_OK;
137 }
138
139 return ret;
140}
141
142/*
143 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
144 * page yet, so it only supports the MSR based communication with the
145 * hypervisor and only the CPUID exit-code.
146 */
147void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
148{
149 unsigned int fn = lower_bits(regs->ax, 32);
150 unsigned long val;
151
152 /* Only CPUID is supported via MSR protocol */
153 if (exit_code != SVM_EXIT_CPUID)
154 goto fail;
155
156 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
157 VMGEXIT();
158 val = sev_es_rd_ghcb_msr();
159 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
160 goto fail;
161 regs->ax = val >> 32;
162
163 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
164 VMGEXIT();
165 val = sev_es_rd_ghcb_msr();
166 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
167 goto fail;
168 regs->bx = val >> 32;
169
170 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
171 VMGEXIT();
172 val = sev_es_rd_ghcb_msr();
173 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
174 goto fail;
175 regs->cx = val >> 32;
176
177 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
178 VMGEXIT();
179 val = sev_es_rd_ghcb_msr();
180 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
181 goto fail;
182 regs->dx = val >> 32;
183
184 /*
185 * This is a VC handler and the #VC is only raised when SEV-ES is
186 * active, which means SEV must be active too. Do sanity checks on the
187 * CPUID results to make sure the hypervisor does not trick the kernel
188 * into the no-sev path. This could map sensitive data unencrypted and
189 * make it accessible to the hypervisor.
190 *
191 * In particular, check for:
192 * - Availability of CPUID leaf 0x8000001f
193 * - SEV CPUID bit.
194 *
195 * The hypervisor might still report the wrong C-bit position, but this
196 * can't be checked here.
197 */
198
199 if (fn == 0x80000000 && (regs->ax < 0x8000001f))
200 /* SEV leaf check */
201 goto fail;
202 else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
203 /* SEV bit */
204 goto fail;
205
206 /* Skip over the CPUID two-byte opcode */
207 regs->ip += 2;
208
209 return;
210
211fail:
212 /* Terminate the guest */
213 sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST);
214}
215
216static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
217 void *src, char *buf,
218 unsigned int data_size,
219 unsigned int count,
220 bool backwards)
221{
222 int i, b = backwards ? -1 : 1;
223 enum es_result ret = ES_OK;
224
225 for (i = 0; i < count; i++) {
226 void *s = src + (i * data_size * b);
227 char *d = buf + (i * data_size);
228
229 ret = vc_read_mem(ctxt, s, d, data_size);
230 if (ret != ES_OK)
231 break;
232 }
233
234 return ret;
235}
236
237static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
238 void *dst, char *buf,
239 unsigned int data_size,
240 unsigned int count,
241 bool backwards)
242{
243 int i, s = backwards ? -1 : 1;
244 enum es_result ret = ES_OK;
245
246 for (i = 0; i < count; i++) {
247 void *d = dst + (i * data_size * s);
248 char *b = buf + (i * data_size);
249
250 ret = vc_write_mem(ctxt, d, b, data_size);
251 if (ret != ES_OK)
252 break;
253 }
254
255 return ret;
256}
257
258#define IOIO_TYPE_STR BIT(2)
259#define IOIO_TYPE_IN 1
260#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
261#define IOIO_TYPE_OUT 0
262#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
263
264#define IOIO_REP BIT(3)
265
266#define IOIO_ADDR_64 BIT(9)
267#define IOIO_ADDR_32 BIT(8)
268#define IOIO_ADDR_16 BIT(7)
269
270#define IOIO_DATA_32 BIT(6)
271#define IOIO_DATA_16 BIT(5)
272#define IOIO_DATA_8 BIT(4)
273
274#define IOIO_SEG_ES (0 << 10)
275#define IOIO_SEG_DS (3 << 10)
276
277static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
278{
279 struct insn *insn = &ctxt->insn;
280 *exitinfo = 0;
281
282 switch (insn->opcode.bytes[0]) {
283 /* INS opcodes */
284 case 0x6c:
285 case 0x6d:
286 *exitinfo |= IOIO_TYPE_INS;
287 *exitinfo |= IOIO_SEG_ES;
288 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
289 break;
290
291 /* OUTS opcodes */
292 case 0x6e:
293 case 0x6f:
294 *exitinfo |= IOIO_TYPE_OUTS;
295 *exitinfo |= IOIO_SEG_DS;
296 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
297 break;
298
299 /* IN immediate opcodes */
300 case 0xe4:
301 case 0xe5:
302 *exitinfo |= IOIO_TYPE_IN;
303 *exitinfo |= (u8)insn->immediate.value << 16;
304 break;
305
306 /* OUT immediate opcodes */
307 case 0xe6:
308 case 0xe7:
309 *exitinfo |= IOIO_TYPE_OUT;
310 *exitinfo |= (u8)insn->immediate.value << 16;
311 break;
312
313 /* IN register opcodes */
314 case 0xec:
315 case 0xed:
316 *exitinfo |= IOIO_TYPE_IN;
317 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
318 break;
319
320 /* OUT register opcodes */
321 case 0xee:
322 case 0xef:
323 *exitinfo |= IOIO_TYPE_OUT;
324 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
325 break;
326
327 default:
328 return ES_DECODE_FAILED;
329 }
330
331 switch (insn->opcode.bytes[0]) {
332 case 0x6c:
333 case 0x6e:
334 case 0xe4:
335 case 0xe6:
336 case 0xec:
337 case 0xee:
338 /* Single byte opcodes */
339 *exitinfo |= IOIO_DATA_8;
340 break;
341 default:
342 /* Length determined by instruction parsing */
343 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
344 : IOIO_DATA_32;
345 }
346 switch (insn->addr_bytes) {
347 case 2:
348 *exitinfo |= IOIO_ADDR_16;
349 break;
350 case 4:
351 *exitinfo |= IOIO_ADDR_32;
352 break;
353 case 8:
354 *exitinfo |= IOIO_ADDR_64;
355 break;
356 }
357
358 if (insn_has_rep_prefix(insn))
359 *exitinfo |= IOIO_REP;
360
361 return ES_OK;
362}
363
364static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
365{
366 struct pt_regs *regs = ctxt->regs;
367 u64 exit_info_1, exit_info_2;
368 enum es_result ret;
369
370 ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
371 if (ret != ES_OK)
372 return ret;
373
374 if (exit_info_1 & IOIO_TYPE_STR) {
375
376 /* (REP) INS/OUTS */
377
378 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
379 unsigned int io_bytes, exit_bytes;
380 unsigned int ghcb_count, op_count;
381 unsigned long es_base;
382 u64 sw_scratch;
383
384 /*
385 * For the string variants with rep prefix the amount of in/out
386 * operations per #VC exception is limited so that the kernel
387 * has a chance to take interrupts and re-schedule while the
388 * instruction is emulated.
389 */
390 io_bytes = (exit_info_1 >> 4) & 0x7;
391 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
392
393 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
394 exit_info_2 = min(op_count, ghcb_count);
395 exit_bytes = exit_info_2 * io_bytes;
396
397 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
398
399 /* Read bytes of OUTS into the shared buffer */
400 if (!(exit_info_1 & IOIO_TYPE_IN)) {
401 ret = vc_insn_string_read(ctxt,
402 (void *)(es_base + regs->si),
403 ghcb->shared_buffer, io_bytes,
404 exit_info_2, df);
405 if (ret)
406 return ret;
407 }
408
409 /*
410 * Issue an VMGEXIT to the HV to consume the bytes from the
411 * shared buffer or to have it write them into the shared buffer
412 * depending on the instruction: OUTS or INS.
413 */
414 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
415 ghcb_set_sw_scratch(ghcb, sw_scratch);
416 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
417 exit_info_1, exit_info_2);
418 if (ret != ES_OK)
419 return ret;
420
421 /* Read bytes from shared buffer into the guest's destination. */
422 if (exit_info_1 & IOIO_TYPE_IN) {
423 ret = vc_insn_string_write(ctxt,
424 (void *)(es_base + regs->di),
425 ghcb->shared_buffer, io_bytes,
426 exit_info_2, df);
427 if (ret)
428 return ret;
429
430 if (df)
431 regs->di -= exit_bytes;
432 else
433 regs->di += exit_bytes;
434 } else {
435 if (df)
436 regs->si -= exit_bytes;
437 else
438 regs->si += exit_bytes;
439 }
440
441 if (exit_info_1 & IOIO_REP)
442 regs->cx -= exit_info_2;
443
444 ret = regs->cx ? ES_RETRY : ES_OK;
445
446 } else {
447
448 /* IN/OUT into/from rAX */
449
450 int bits = (exit_info_1 & 0x70) >> 1;
451 u64 rax = 0;
452
453 if (!(exit_info_1 & IOIO_TYPE_IN))
454 rax = lower_bits(regs->ax, bits);
455
456 ghcb_set_rax(ghcb, rax);
457
458 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
459 if (ret != ES_OK)
460 return ret;
461
462 if (exit_info_1 & IOIO_TYPE_IN) {
463 if (!ghcb_rax_is_valid(ghcb))
464 return ES_VMM_ERROR;
465 regs->ax = lower_bits(ghcb->save.rax, bits);
466 }
467 }
468
469 return ret;
470}
471
472static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
473 struct es_em_ctxt *ctxt)
474{
475 struct pt_regs *regs = ctxt->regs;
476 u32 cr4 = native_read_cr4();
477 enum es_result ret;
478
479 ghcb_set_rax(ghcb, regs->ax);
480 ghcb_set_rcx(ghcb, regs->cx);
481
482 if (cr4 & X86_CR4_OSXSAVE)
483 /* Safe to read xcr0 */
484 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
485 else
486 /* xgetbv will cause #GP - use reset value for xcr0 */
487 ghcb_set_xcr0(ghcb, 1);
488
489 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
490 if (ret != ES_OK)
491 return ret;
492
493 if (!(ghcb_rax_is_valid(ghcb) &&
494 ghcb_rbx_is_valid(ghcb) &&
495 ghcb_rcx_is_valid(ghcb) &&
496 ghcb_rdx_is_valid(ghcb)))
497 return ES_VMM_ERROR;
498
499 regs->ax = ghcb->save.rax;
500 regs->bx = ghcb->save.rbx;
501 regs->cx = ghcb->save.rcx;
502 regs->dx = ghcb->save.rdx;
503
504 return ES_OK;
505}
506
507static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
508 struct es_em_ctxt *ctxt,
509 unsigned long exit_code)
510{
511 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
512 enum es_result ret;
513
514 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
515 if (ret != ES_OK)
516 return ret;
517
518 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
519 (!rdtscp || ghcb_rcx_is_valid(ghcb))))
520 return ES_VMM_ERROR;
521
522 ctxt->regs->ax = ghcb->save.rax;
523 ctxt->regs->dx = ghcb->save.rdx;
524 if (rdtscp)
525 ctxt->regs->cx = ghcb->save.rcx;
526
527 return ES_OK;
528}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 *
7 * This file is not compiled stand-alone. It contains code shared
8 * between the pre-decompression boot code and the running Linux kernel
9 * and is included directly into both code-bases.
10 */
11
12#ifndef __BOOT_COMPRESSED
13#define error(v) pr_err(v)
14#define has_cpuflag(f) boot_cpu_has(f)
15#else
16#undef WARN
17#define WARN(condition, format...) (!!(condition))
18#endif
19
20/* I/O parameters for CPUID-related helpers */
21struct cpuid_leaf {
22 u32 fn;
23 u32 subfn;
24 u32 eax;
25 u32 ebx;
26 u32 ecx;
27 u32 edx;
28};
29
30/*
31 * Individual entries of the SNP CPUID table, as defined by the SNP
32 * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
33 */
34struct snp_cpuid_fn {
35 u32 eax_in;
36 u32 ecx_in;
37 u64 xcr0_in;
38 u64 xss_in;
39 u32 eax;
40 u32 ebx;
41 u32 ecx;
42 u32 edx;
43 u64 __reserved;
44} __packed;
45
46/*
47 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
48 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
49 * of 64 entries per CPUID table.
50 */
51#define SNP_CPUID_COUNT_MAX 64
52
53struct snp_cpuid_table {
54 u32 count;
55 u32 __reserved1;
56 u64 __reserved2;
57 struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
58} __packed;
59
60/*
61 * Since feature negotiation related variables are set early in the boot
62 * process they must reside in the .data section so as not to be zeroed
63 * out when the .bss section is later cleared.
64 *
65 * GHCB protocol version negotiated with the hypervisor.
66 */
67static u16 ghcb_version __ro_after_init;
68
69/* Copy of the SNP firmware's CPUID page. */
70static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
71
72/*
73 * These will be initialized based on CPUID table so that non-present
74 * all-zero leaves (for sparse tables) can be differentiated from
75 * invalid/out-of-range leaves. This is needed since all-zero leaves
76 * still need to be post-processed.
77 */
78static u32 cpuid_std_range_max __ro_after_init;
79static u32 cpuid_hyp_range_max __ro_after_init;
80static u32 cpuid_ext_range_max __ro_after_init;
81
82static bool __init sev_es_check_cpu_features(void)
83{
84 if (!has_cpuflag(X86_FEATURE_RDRAND)) {
85 error("RDRAND instruction not supported - no trusted source of randomness available\n");
86 return false;
87 }
88
89 return true;
90}
91
92static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason)
93{
94 u64 val = GHCB_MSR_TERM_REQ;
95
96 /* Tell the hypervisor what went wrong. */
97 val |= GHCB_SEV_TERM_REASON(set, reason);
98
99 /* Request Guest Termination from Hypervisor */
100 sev_es_wr_ghcb_msr(val);
101 VMGEXIT();
102
103 while (true)
104 asm volatile("hlt\n" : : : "memory");
105}
106
107/*
108 * The hypervisor features are available from GHCB version 2 onward.
109 */
110static u64 get_hv_features(void)
111{
112 u64 val;
113
114 if (ghcb_version < 2)
115 return 0;
116
117 sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
118 VMGEXIT();
119
120 val = sev_es_rd_ghcb_msr();
121 if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
122 return 0;
123
124 return GHCB_MSR_HV_FT_RESP_VAL(val);
125}
126
127static void snp_register_ghcb_early(unsigned long paddr)
128{
129 unsigned long pfn = paddr >> PAGE_SHIFT;
130 u64 val;
131
132 sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
133 VMGEXIT();
134
135 val = sev_es_rd_ghcb_msr();
136
137 /* If the response GPA is not ours then abort the guest */
138 if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
139 (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
140 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
141}
142
143static bool sev_es_negotiate_protocol(void)
144{
145 u64 val;
146
147 /* Do the GHCB protocol version negotiation */
148 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
149 VMGEXIT();
150 val = sev_es_rd_ghcb_msr();
151
152 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
153 return false;
154
155 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
156 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
157 return false;
158
159 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
160
161 return true;
162}
163
164static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
165{
166 ghcb->save.sw_exit_code = 0;
167 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
168}
169
170static bool vc_decoding_needed(unsigned long exit_code)
171{
172 /* Exceptions don't require to decode the instruction */
173 return !(exit_code >= SVM_EXIT_EXCP_BASE &&
174 exit_code <= SVM_EXIT_LAST_EXCP);
175}
176
177static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
178 struct pt_regs *regs,
179 unsigned long exit_code)
180{
181 enum es_result ret = ES_OK;
182
183 memset(ctxt, 0, sizeof(*ctxt));
184 ctxt->regs = regs;
185
186 if (vc_decoding_needed(exit_code))
187 ret = vc_decode_insn(ctxt);
188
189 return ret;
190}
191
192static void vc_finish_insn(struct es_em_ctxt *ctxt)
193{
194 ctxt->regs->ip += ctxt->insn.length;
195}
196
197static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
198{
199 u32 ret;
200
201 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
202 if (!ret)
203 return ES_OK;
204
205 if (ret == 1) {
206 u64 info = ghcb->save.sw_exit_info_2;
207 unsigned long v = info & SVM_EVTINJ_VEC_MASK;
208
209 /* Check if exception information from hypervisor is sane. */
210 if ((info & SVM_EVTINJ_VALID) &&
211 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
212 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
213 ctxt->fi.vector = v;
214
215 if (info & SVM_EVTINJ_VALID_ERR)
216 ctxt->fi.error_code = info >> 32;
217
218 return ES_EXCEPTION;
219 }
220 }
221
222 return ES_VMM_ERROR;
223}
224
225static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
226 struct es_em_ctxt *ctxt,
227 u64 exit_code, u64 exit_info_1,
228 u64 exit_info_2)
229{
230 /* Fill in protocol and format specifiers */
231 ghcb->protocol_version = ghcb_version;
232 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
233
234 ghcb_set_sw_exit_code(ghcb, exit_code);
235 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
236 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
237
238 sev_es_wr_ghcb_msr(__pa(ghcb));
239 VMGEXIT();
240
241 return verify_exception_info(ghcb, ctxt);
242}
243
244static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
245{
246 u64 val;
247
248 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
249 VMGEXIT();
250 val = sev_es_rd_ghcb_msr();
251 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
252 return -EIO;
253
254 *reg = (val >> 32);
255
256 return 0;
257}
258
259static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf)
260{
261 int ret;
262
263 /*
264 * MSR protocol does not support fetching non-zero subfunctions, but is
265 * sufficient to handle current early-boot cases. Should that change,
266 * make sure to report an error rather than ignoring the index and
267 * grabbing random values. If this issue arises in the future, handling
268 * can be added here to use GHCB-page protocol for cases that occur late
269 * enough in boot that GHCB page is available.
270 */
271 if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
272 return -EINVAL;
273
274 ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
275 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
276 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
277 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
278
279 return ret;
280}
281
282static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
283{
284 u32 cr4 = native_read_cr4();
285 int ret;
286
287 ghcb_set_rax(ghcb, leaf->fn);
288 ghcb_set_rcx(ghcb, leaf->subfn);
289
290 if (cr4 & X86_CR4_OSXSAVE)
291 /* Safe to read xcr0 */
292 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
293 else
294 /* xgetbv will cause #UD - use reset value for xcr0 */
295 ghcb_set_xcr0(ghcb, 1);
296
297 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
298 if (ret != ES_OK)
299 return ret;
300
301 if (!(ghcb_rax_is_valid(ghcb) &&
302 ghcb_rbx_is_valid(ghcb) &&
303 ghcb_rcx_is_valid(ghcb) &&
304 ghcb_rdx_is_valid(ghcb)))
305 return ES_VMM_ERROR;
306
307 leaf->eax = ghcb->save.rax;
308 leaf->ebx = ghcb->save.rbx;
309 leaf->ecx = ghcb->save.rcx;
310 leaf->edx = ghcb->save.rdx;
311
312 return ES_OK;
313}
314
315static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
316{
317 return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf)
318 : __sev_cpuid_hv_msr(leaf);
319}
320
321/*
322 * This may be called early while still running on the initial identity
323 * mapping. Use RIP-relative addressing to obtain the correct address
324 * while running with the initial identity mapping as well as the
325 * switch-over to kernel virtual addresses later.
326 */
327static const struct snp_cpuid_table *snp_cpuid_get_table(void)
328{
329 void *ptr;
330
331 asm ("lea cpuid_table_copy(%%rip), %0"
332 : "=r" (ptr)
333 : "p" (&cpuid_table_copy));
334
335 return ptr;
336}
337
338/*
339 * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
340 * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
341 * and 1 based on the corresponding features enabled by a particular
342 * combination of XCR0 and XSS registers so that a guest can look up the
343 * version corresponding to the features currently enabled in its XCR0/XSS
344 * registers. The only values that differ between these versions/table
345 * entries is the enabled XSAVE area size advertised via EBX.
346 *
347 * While hypervisors may choose to make use of this support, it is more
348 * robust/secure for a guest to simply find the entry corresponding to the
349 * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
350 * XSAVE area size using subfunctions 2 through 64, as documented in APM
351 * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
352 *
353 * Since base/legacy XSAVE area size is documented as 0x240, use that value
354 * directly rather than relying on the base size in the CPUID table.
355 *
356 * Return: XSAVE area size on success, 0 otherwise.
357 */
358static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
359{
360 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
361 u64 xfeatures_found = 0;
362 u32 xsave_size = 0x240;
363 int i;
364
365 for (i = 0; i < cpuid_table->count; i++) {
366 const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
367
368 if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
369 continue;
370 if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
371 continue;
372 if (xfeatures_found & (BIT_ULL(e->ecx_in)))
373 continue;
374
375 xfeatures_found |= (BIT_ULL(e->ecx_in));
376
377 if (compacted)
378 xsave_size += e->eax;
379 else
380 xsave_size = max(xsave_size, e->eax + e->ebx);
381 }
382
383 /*
384 * Either the guest set unsupported XCR0/XSS bits, or the corresponding
385 * entries in the CPUID table were not present. This is not a valid
386 * state to be in.
387 */
388 if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
389 return 0;
390
391 return xsave_size;
392}
393
394static bool
395snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
396{
397 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
398 int i;
399
400 for (i = 0; i < cpuid_table->count; i++) {
401 const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
402
403 if (e->eax_in != leaf->fn)
404 continue;
405
406 if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
407 continue;
408
409 /*
410 * For 0xD subfunctions 0 and 1, only use the entry corresponding
411 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
412 * See the comments above snp_cpuid_calc_xsave_size() for more
413 * details.
414 */
415 if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
416 if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
417 continue;
418
419 leaf->eax = e->eax;
420 leaf->ebx = e->ebx;
421 leaf->ecx = e->ecx;
422 leaf->edx = e->edx;
423
424 return true;
425 }
426
427 return false;
428}
429
430static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
431{
432 if (sev_cpuid_hv(ghcb, ctxt, leaf))
433 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
434}
435
436static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
437 struct cpuid_leaf *leaf)
438{
439 struct cpuid_leaf leaf_hv = *leaf;
440
441 switch (leaf->fn) {
442 case 0x1:
443 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
444
445 /* initial APIC ID */
446 leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
447 /* APIC enabled bit */
448 leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
449
450 /* OSXSAVE enabled bit */
451 if (native_read_cr4() & X86_CR4_OSXSAVE)
452 leaf->ecx |= BIT(27);
453 break;
454 case 0x7:
455 /* OSPKE enabled bit */
456 leaf->ecx &= ~BIT(4);
457 if (native_read_cr4() & X86_CR4_PKE)
458 leaf->ecx |= BIT(4);
459 break;
460 case 0xB:
461 leaf_hv.subfn = 0;
462 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
463
464 /* extended APIC ID */
465 leaf->edx = leaf_hv.edx;
466 break;
467 case 0xD: {
468 bool compacted = false;
469 u64 xcr0 = 1, xss = 0;
470 u32 xsave_size;
471
472 if (leaf->subfn != 0 && leaf->subfn != 1)
473 return 0;
474
475 if (native_read_cr4() & X86_CR4_OSXSAVE)
476 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
477 if (leaf->subfn == 1) {
478 /* Get XSS value if XSAVES is enabled. */
479 if (leaf->eax & BIT(3)) {
480 unsigned long lo, hi;
481
482 asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
483 : "c" (MSR_IA32_XSS));
484 xss = (hi << 32) | lo;
485 }
486
487 /*
488 * The PPR and APM aren't clear on what size should be
489 * encoded in 0xD:0x1:EBX when compaction is not enabled
490 * by either XSAVEC (feature bit 1) or XSAVES (feature
491 * bit 3) since SNP-capable hardware has these feature
492 * bits fixed as 1. KVM sets it to 0 in this case, but
493 * to avoid this becoming an issue it's safer to simply
494 * treat this as unsupported for SNP guests.
495 */
496 if (!(leaf->eax & (BIT(1) | BIT(3))))
497 return -EINVAL;
498
499 compacted = true;
500 }
501
502 xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
503 if (!xsave_size)
504 return -EINVAL;
505
506 leaf->ebx = xsave_size;
507 }
508 break;
509 case 0x8000001E:
510 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
511
512 /* extended APIC ID */
513 leaf->eax = leaf_hv.eax;
514 /* compute ID */
515 leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
516 /* node ID */
517 leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
518 break;
519 default:
520 /* No fix-ups needed, use values as-is. */
521 break;
522 }
523
524 return 0;
525}
526
527/*
528 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
529 * should be treated as fatal by caller.
530 */
531static int snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
532{
533 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
534
535 if (!cpuid_table->count)
536 return -EOPNOTSUPP;
537
538 if (!snp_cpuid_get_validated_func(leaf)) {
539 /*
540 * Some hypervisors will avoid keeping track of CPUID entries
541 * where all values are zero, since they can be handled the
542 * same as out-of-range values (all-zero). This is useful here
543 * as well as it allows virtually all guest configurations to
544 * work using a single SNP CPUID table.
545 *
546 * To allow for this, there is a need to distinguish between
547 * out-of-range entries and in-range zero entries, since the
548 * CPUID table entries are only a template that may need to be
549 * augmented with additional values for things like
550 * CPU-specific information during post-processing. So if it's
551 * not in the table, set the values to zero. Then, if they are
552 * within a valid CPUID range, proceed with post-processing
553 * using zeros as the initial values. Otherwise, skip
554 * post-processing and just return zeros immediately.
555 */
556 leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
557
558 /* Skip post-processing for out-of-range zero leafs. */
559 if (!(leaf->fn <= cpuid_std_range_max ||
560 (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) ||
561 (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))
562 return 0;
563 }
564
565 return snp_cpuid_postprocess(ghcb, ctxt, leaf);
566}
567
568/*
569 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
570 * page yet, so it only supports the MSR based communication with the
571 * hypervisor and only the CPUID exit-code.
572 */
573void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
574{
575 unsigned int subfn = lower_bits(regs->cx, 32);
576 unsigned int fn = lower_bits(regs->ax, 32);
577 struct cpuid_leaf leaf;
578 int ret;
579
580 /* Only CPUID is supported via MSR protocol */
581 if (exit_code != SVM_EXIT_CPUID)
582 goto fail;
583
584 leaf.fn = fn;
585 leaf.subfn = subfn;
586
587 ret = snp_cpuid(NULL, NULL, &leaf);
588 if (!ret)
589 goto cpuid_done;
590
591 if (ret != -EOPNOTSUPP)
592 goto fail;
593
594 if (__sev_cpuid_hv_msr(&leaf))
595 goto fail;
596
597cpuid_done:
598 regs->ax = leaf.eax;
599 regs->bx = leaf.ebx;
600 regs->cx = leaf.ecx;
601 regs->dx = leaf.edx;
602
603 /*
604 * This is a VC handler and the #VC is only raised when SEV-ES is
605 * active, which means SEV must be active too. Do sanity checks on the
606 * CPUID results to make sure the hypervisor does not trick the kernel
607 * into the no-sev path. This could map sensitive data unencrypted and
608 * make it accessible to the hypervisor.
609 *
610 * In particular, check for:
611 * - Availability of CPUID leaf 0x8000001f
612 * - SEV CPUID bit.
613 *
614 * The hypervisor might still report the wrong C-bit position, but this
615 * can't be checked here.
616 */
617
618 if (fn == 0x80000000 && (regs->ax < 0x8000001f))
619 /* SEV leaf check */
620 goto fail;
621 else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
622 /* SEV bit */
623 goto fail;
624
625 /* Skip over the CPUID two-byte opcode */
626 regs->ip += 2;
627
628 return;
629
630fail:
631 /* Terminate the guest */
632 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
633}
634
635static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
636 unsigned long address,
637 bool write)
638{
639 if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
640 ctxt->fi.vector = X86_TRAP_PF;
641 ctxt->fi.error_code = X86_PF_USER;
642 ctxt->fi.cr2 = address;
643 if (write)
644 ctxt->fi.error_code |= X86_PF_WRITE;
645
646 return ES_EXCEPTION;
647 }
648
649 return ES_OK;
650}
651
652static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
653 void *src, char *buf,
654 unsigned int data_size,
655 unsigned int count,
656 bool backwards)
657{
658 int i, b = backwards ? -1 : 1;
659 unsigned long address = (unsigned long)src;
660 enum es_result ret;
661
662 ret = vc_insn_string_check(ctxt, address, false);
663 if (ret != ES_OK)
664 return ret;
665
666 for (i = 0; i < count; i++) {
667 void *s = src + (i * data_size * b);
668 char *d = buf + (i * data_size);
669
670 ret = vc_read_mem(ctxt, s, d, data_size);
671 if (ret != ES_OK)
672 break;
673 }
674
675 return ret;
676}
677
678static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
679 void *dst, char *buf,
680 unsigned int data_size,
681 unsigned int count,
682 bool backwards)
683{
684 int i, s = backwards ? -1 : 1;
685 unsigned long address = (unsigned long)dst;
686 enum es_result ret;
687
688 ret = vc_insn_string_check(ctxt, address, true);
689 if (ret != ES_OK)
690 return ret;
691
692 for (i = 0; i < count; i++) {
693 void *d = dst + (i * data_size * s);
694 char *b = buf + (i * data_size);
695
696 ret = vc_write_mem(ctxt, d, b, data_size);
697 if (ret != ES_OK)
698 break;
699 }
700
701 return ret;
702}
703
704#define IOIO_TYPE_STR BIT(2)
705#define IOIO_TYPE_IN 1
706#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
707#define IOIO_TYPE_OUT 0
708#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
709
710#define IOIO_REP BIT(3)
711
712#define IOIO_ADDR_64 BIT(9)
713#define IOIO_ADDR_32 BIT(8)
714#define IOIO_ADDR_16 BIT(7)
715
716#define IOIO_DATA_32 BIT(6)
717#define IOIO_DATA_16 BIT(5)
718#define IOIO_DATA_8 BIT(4)
719
720#define IOIO_SEG_ES (0 << 10)
721#define IOIO_SEG_DS (3 << 10)
722
723static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
724{
725 struct insn *insn = &ctxt->insn;
726 size_t size;
727 u64 port;
728
729 *exitinfo = 0;
730
731 switch (insn->opcode.bytes[0]) {
732 /* INS opcodes */
733 case 0x6c:
734 case 0x6d:
735 *exitinfo |= IOIO_TYPE_INS;
736 *exitinfo |= IOIO_SEG_ES;
737 port = ctxt->regs->dx & 0xffff;
738 break;
739
740 /* OUTS opcodes */
741 case 0x6e:
742 case 0x6f:
743 *exitinfo |= IOIO_TYPE_OUTS;
744 *exitinfo |= IOIO_SEG_DS;
745 port = ctxt->regs->dx & 0xffff;
746 break;
747
748 /* IN immediate opcodes */
749 case 0xe4:
750 case 0xe5:
751 *exitinfo |= IOIO_TYPE_IN;
752 port = (u8)insn->immediate.value & 0xffff;
753 break;
754
755 /* OUT immediate opcodes */
756 case 0xe6:
757 case 0xe7:
758 *exitinfo |= IOIO_TYPE_OUT;
759 port = (u8)insn->immediate.value & 0xffff;
760 break;
761
762 /* IN register opcodes */
763 case 0xec:
764 case 0xed:
765 *exitinfo |= IOIO_TYPE_IN;
766 port = ctxt->regs->dx & 0xffff;
767 break;
768
769 /* OUT register opcodes */
770 case 0xee:
771 case 0xef:
772 *exitinfo |= IOIO_TYPE_OUT;
773 port = ctxt->regs->dx & 0xffff;
774 break;
775
776 default:
777 return ES_DECODE_FAILED;
778 }
779
780 *exitinfo |= port << 16;
781
782 switch (insn->opcode.bytes[0]) {
783 case 0x6c:
784 case 0x6e:
785 case 0xe4:
786 case 0xe6:
787 case 0xec:
788 case 0xee:
789 /* Single byte opcodes */
790 *exitinfo |= IOIO_DATA_8;
791 size = 1;
792 break;
793 default:
794 /* Length determined by instruction parsing */
795 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
796 : IOIO_DATA_32;
797 size = (insn->opnd_bytes == 2) ? 2 : 4;
798 }
799
800 switch (insn->addr_bytes) {
801 case 2:
802 *exitinfo |= IOIO_ADDR_16;
803 break;
804 case 4:
805 *exitinfo |= IOIO_ADDR_32;
806 break;
807 case 8:
808 *exitinfo |= IOIO_ADDR_64;
809 break;
810 }
811
812 if (insn_has_rep_prefix(insn))
813 *exitinfo |= IOIO_REP;
814
815 return vc_ioio_check(ctxt, (u16)port, size);
816}
817
818static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
819{
820 struct pt_regs *regs = ctxt->regs;
821 u64 exit_info_1, exit_info_2;
822 enum es_result ret;
823
824 ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
825 if (ret != ES_OK)
826 return ret;
827
828 if (exit_info_1 & IOIO_TYPE_STR) {
829
830 /* (REP) INS/OUTS */
831
832 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
833 unsigned int io_bytes, exit_bytes;
834 unsigned int ghcb_count, op_count;
835 unsigned long es_base;
836 u64 sw_scratch;
837
838 /*
839 * For the string variants with rep prefix the amount of in/out
840 * operations per #VC exception is limited so that the kernel
841 * has a chance to take interrupts and re-schedule while the
842 * instruction is emulated.
843 */
844 io_bytes = (exit_info_1 >> 4) & 0x7;
845 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
846
847 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
848 exit_info_2 = min(op_count, ghcb_count);
849 exit_bytes = exit_info_2 * io_bytes;
850
851 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
852
853 /* Read bytes of OUTS into the shared buffer */
854 if (!(exit_info_1 & IOIO_TYPE_IN)) {
855 ret = vc_insn_string_read(ctxt,
856 (void *)(es_base + regs->si),
857 ghcb->shared_buffer, io_bytes,
858 exit_info_2, df);
859 if (ret)
860 return ret;
861 }
862
863 /*
864 * Issue an VMGEXIT to the HV to consume the bytes from the
865 * shared buffer or to have it write them into the shared buffer
866 * depending on the instruction: OUTS or INS.
867 */
868 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
869 ghcb_set_sw_scratch(ghcb, sw_scratch);
870 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
871 exit_info_1, exit_info_2);
872 if (ret != ES_OK)
873 return ret;
874
875 /* Read bytes from shared buffer into the guest's destination. */
876 if (exit_info_1 & IOIO_TYPE_IN) {
877 ret = vc_insn_string_write(ctxt,
878 (void *)(es_base + regs->di),
879 ghcb->shared_buffer, io_bytes,
880 exit_info_2, df);
881 if (ret)
882 return ret;
883
884 if (df)
885 regs->di -= exit_bytes;
886 else
887 regs->di += exit_bytes;
888 } else {
889 if (df)
890 regs->si -= exit_bytes;
891 else
892 regs->si += exit_bytes;
893 }
894
895 if (exit_info_1 & IOIO_REP)
896 regs->cx -= exit_info_2;
897
898 ret = regs->cx ? ES_RETRY : ES_OK;
899
900 } else {
901
902 /* IN/OUT into/from rAX */
903
904 int bits = (exit_info_1 & 0x70) >> 1;
905 u64 rax = 0;
906
907 if (!(exit_info_1 & IOIO_TYPE_IN))
908 rax = lower_bits(regs->ax, bits);
909
910 ghcb_set_rax(ghcb, rax);
911
912 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
913 if (ret != ES_OK)
914 return ret;
915
916 if (exit_info_1 & IOIO_TYPE_IN) {
917 if (!ghcb_rax_is_valid(ghcb))
918 return ES_VMM_ERROR;
919 regs->ax = lower_bits(ghcb->save.rax, bits);
920 }
921 }
922
923 return ret;
924}
925
926static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
927{
928 struct pt_regs *regs = ctxt->regs;
929 struct cpuid_leaf leaf;
930 int ret;
931
932 leaf.fn = regs->ax;
933 leaf.subfn = regs->cx;
934 ret = snp_cpuid(ghcb, ctxt, &leaf);
935 if (!ret) {
936 regs->ax = leaf.eax;
937 regs->bx = leaf.ebx;
938 regs->cx = leaf.ecx;
939 regs->dx = leaf.edx;
940 }
941
942 return ret;
943}
944
945static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
946 struct es_em_ctxt *ctxt)
947{
948 struct pt_regs *regs = ctxt->regs;
949 u32 cr4 = native_read_cr4();
950 enum es_result ret;
951 int snp_cpuid_ret;
952
953 snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
954 if (!snp_cpuid_ret)
955 return ES_OK;
956 if (snp_cpuid_ret != -EOPNOTSUPP)
957 return ES_VMM_ERROR;
958
959 ghcb_set_rax(ghcb, regs->ax);
960 ghcb_set_rcx(ghcb, regs->cx);
961
962 if (cr4 & X86_CR4_OSXSAVE)
963 /* Safe to read xcr0 */
964 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
965 else
966 /* xgetbv will cause #GP - use reset value for xcr0 */
967 ghcb_set_xcr0(ghcb, 1);
968
969 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
970 if (ret != ES_OK)
971 return ret;
972
973 if (!(ghcb_rax_is_valid(ghcb) &&
974 ghcb_rbx_is_valid(ghcb) &&
975 ghcb_rcx_is_valid(ghcb) &&
976 ghcb_rdx_is_valid(ghcb)))
977 return ES_VMM_ERROR;
978
979 regs->ax = ghcb->save.rax;
980 regs->bx = ghcb->save.rbx;
981 regs->cx = ghcb->save.rcx;
982 regs->dx = ghcb->save.rdx;
983
984 return ES_OK;
985}
986
987static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
988 struct es_em_ctxt *ctxt,
989 unsigned long exit_code)
990{
991 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
992 enum es_result ret;
993
994 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
995 if (ret != ES_OK)
996 return ret;
997
998 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
999 (!rdtscp || ghcb_rcx_is_valid(ghcb))))
1000 return ES_VMM_ERROR;
1001
1002 ctxt->regs->ax = ghcb->save.rax;
1003 ctxt->regs->dx = ghcb->save.rdx;
1004 if (rdtscp)
1005 ctxt->regs->cx = ghcb->save.rcx;
1006
1007 return ES_OK;
1008}
1009
1010struct cc_setup_data {
1011 struct setup_data header;
1012 u32 cc_blob_address;
1013};
1014
1015/*
1016 * Search for a Confidential Computing blob passed in as a setup_data entry
1017 * via the Linux Boot Protocol.
1018 */
1019static struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
1020{
1021 struct cc_setup_data *sd = NULL;
1022 struct setup_data *hdr;
1023
1024 hdr = (struct setup_data *)bp->hdr.setup_data;
1025
1026 while (hdr) {
1027 if (hdr->type == SETUP_CC_BLOB) {
1028 sd = (struct cc_setup_data *)hdr;
1029 return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
1030 }
1031 hdr = (struct setup_data *)hdr->next;
1032 }
1033
1034 return NULL;
1035}
1036
1037/*
1038 * Initialize the kernel's copy of the SNP CPUID table, and set up the
1039 * pointer that will be used to access it.
1040 *
1041 * Maintaining a direct mapping of the SNP CPUID table used by firmware would
1042 * be possible as an alternative, but the approach is brittle since the
1043 * mapping needs to be updated in sync with all the changes to virtual memory
1044 * layout and related mapping facilities throughout the boot process.
1045 */
1046static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
1047{
1048 const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
1049 int i;
1050
1051 if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
1052 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
1053
1054 cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
1055 if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
1056 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
1057
1058 cpuid_table = snp_cpuid_get_table();
1059 memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
1060
1061 /* Initialize CPUID ranges for range-checking. */
1062 for (i = 0; i < cpuid_table->count; i++) {
1063 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
1064
1065 if (fn->eax_in == 0x0)
1066 cpuid_std_range_max = fn->eax;
1067 else if (fn->eax_in == 0x40000000)
1068 cpuid_hyp_range_max = fn->eax;
1069 else if (fn->eax_in == 0x80000000)
1070 cpuid_ext_range_max = fn->eax;
1071 }
1072}
1073
1074static void pvalidate_pages(struct snp_psc_desc *desc)
1075{
1076 struct psc_entry *e;
1077 unsigned long vaddr;
1078 unsigned int size;
1079 unsigned int i;
1080 bool validate;
1081 int rc;
1082
1083 for (i = 0; i <= desc->hdr.end_entry; i++) {
1084 e = &desc->entries[i];
1085
1086 vaddr = (unsigned long)pfn_to_kaddr(e->gfn);
1087 size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
1088 validate = e->operation == SNP_PAGE_STATE_PRIVATE;
1089
1090 rc = pvalidate(vaddr, size, validate);
1091 if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
1092 unsigned long vaddr_end = vaddr + PMD_SIZE;
1093
1094 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) {
1095 rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
1096 if (rc)
1097 break;
1098 }
1099 }
1100
1101 if (rc) {
1102 WARN(1, "Failed to validate address 0x%lx ret %d", vaddr, rc);
1103 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
1104 }
1105 }
1106}
1107
1108static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
1109{
1110 int cur_entry, end_entry, ret = 0;
1111 struct snp_psc_desc *data;
1112 struct es_em_ctxt ctxt;
1113
1114 vc_ghcb_invalidate(ghcb);
1115
1116 /* Copy the input desc into GHCB shared buffer */
1117 data = (struct snp_psc_desc *)ghcb->shared_buffer;
1118 memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
1119
1120 /*
1121 * As per the GHCB specification, the hypervisor can resume the guest
1122 * before processing all the entries. Check whether all the entries
1123 * are processed. If not, then keep retrying. Note, the hypervisor
1124 * will update the data memory directly to indicate the status, so
1125 * reference the data->hdr everywhere.
1126 *
1127 * The strategy here is to wait for the hypervisor to change the page
1128 * state in the RMP table before guest accesses the memory pages. If the
1129 * page state change was not successful, then later memory access will
1130 * result in a crash.
1131 */
1132 cur_entry = data->hdr.cur_entry;
1133 end_entry = data->hdr.end_entry;
1134
1135 while (data->hdr.cur_entry <= data->hdr.end_entry) {
1136 ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
1137
1138 /* This will advance the shared buffer data points to. */
1139 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
1140
1141 /*
1142 * Page State Change VMGEXIT can pass error code through
1143 * exit_info_2.
1144 */
1145 if (WARN(ret || ghcb->save.sw_exit_info_2,
1146 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
1147 ret, ghcb->save.sw_exit_info_2)) {
1148 ret = 1;
1149 goto out;
1150 }
1151
1152 /* Verify that reserved bit is not set */
1153 if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
1154 ret = 1;
1155 goto out;
1156 }
1157
1158 /*
1159 * Sanity check that entry processing is not going backwards.
1160 * This will happen only if hypervisor is tricking us.
1161 */
1162 if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
1163"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
1164 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
1165 ret = 1;
1166 goto out;
1167 }
1168 }
1169
1170out:
1171 return ret;
1172}