1#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   2
   3#include <linux/kvm_host.h>
   4
   5#include "irq.h"
   6#include "mmu.h"
   7#include "kvm_cache_regs.h"
   8#include "x86.h"
   9#include "smm.h"
  10#include "cpuid.h"
  11#include "pmu.h"
  12
  13#include <linux/module.h>
  14#include <linux/mod_devicetable.h>
  15#include <linux/kernel.h>
  16#include <linux/vmalloc.h>
  17#include <linux/highmem.h>
  18#include <linux/amd-iommu.h>
  19#include <linux/sched.h>
  20#include <linux/trace_events.h>
  21#include <linux/slab.h>
  22#include <linux/hashtable.h>
  23#include <linux/objtool.h>
  24#include <linux/psp-sev.h>
  25#include <linux/file.h>
  26#include <linux/pagemap.h>
  27#include <linux/swap.h>
  28#include <linux/rwsem.h>
  29#include <linux/cc_platform.h>
  30#include <linux/smp.h>
  31
  32#include <asm/apic.h>
  33#include <asm/perf_event.h>
  34#include <asm/tlbflush.h>
  35#include <asm/desc.h>
  36#include <asm/debugreg.h>
  37#include <asm/kvm_para.h>
  38#include <asm/irq_remapping.h>
  39#include <asm/spec-ctrl.h>
  40#include <asm/cpu_device_id.h>
  41#include <asm/traps.h>
  42#include <asm/reboot.h>
  43#include <asm/fpu/api.h>
  44
  45#include <trace/events/ipi.h>
  46
  47#include "trace.h"
  48
  49#include "svm.h"
  50#include "svm_ops.h"
  51
  52#include "kvm_onhyperv.h"
  53#include "svm_onhyperv.h"
  54
  55MODULE_AUTHOR("Qumranet");
  56MODULE_DESCRIPTION("KVM support for SVM (AMD-V) extensions");
  57MODULE_LICENSE("GPL");
  58
  59#ifdef MODULE
  60static const struct x86_cpu_id svm_cpu_id[] = {
  61	X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
  62	{}
  63};
  64MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  65#endif
  66
  67#define SEG_TYPE_LDT 2
  68#define SEG_TYPE_BUSY_TSS16 3
  69
  70static bool erratum_383_found __read_mostly;
  71
  72u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
  73
  74/*
  75 * Set osvw_len to higher value when updated Revision Guides
  76 * are published and we know what the new status bits are
  77 */
  78static uint64_t osvw_len = 4, osvw_status;
  79
  80static DEFINE_PER_CPU(u64, current_tsc_ratio);
  81
  82#define X2APIC_MSR(x)	(APIC_BASE_MSR + (x >> 4))
  83
  84static const struct svm_direct_access_msrs {
  85	u32 index;   /* Index of the MSR */
  86	bool always; /* True if intercept is initially cleared */
  87} direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
  88	{ .index = MSR_STAR,				.always = true  },
  89	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
  90	{ .index = MSR_IA32_SYSENTER_EIP,		.always = false },
  91	{ .index = MSR_IA32_SYSENTER_ESP,		.always = false },
  92#ifdef CONFIG_X86_64
  93	{ .index = MSR_GS_BASE,				.always = true  },
  94	{ .index = MSR_FS_BASE,				.always = true  },
  95	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
  96	{ .index = MSR_LSTAR,				.always = true  },
  97	{ .index = MSR_CSTAR,				.always = true  },
  98	{ .index = MSR_SYSCALL_MASK,			.always = true  },
  99#endif
 100	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
 101	{ .index = MSR_IA32_PRED_CMD,			.always = false },
 102	{ .index = MSR_IA32_FLUSH_CMD,			.always = false },
 103	{ .index = MSR_IA32_DEBUGCTLMSR,		.always = false },
 104	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
 105	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
 106	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
 107	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
 108	{ .index = MSR_IA32_XSS,			.always = false },
 109	{ .index = MSR_EFER,				.always = false },
 110	{ .index = MSR_IA32_CR_PAT,			.always = false },
 111	{ .index = MSR_AMD64_SEV_ES_GHCB,		.always = true  },
 112	{ .index = MSR_TSC_AUX,				.always = false },
 113	{ .index = X2APIC_MSR(APIC_ID),			.always = false },
 114	{ .index = X2APIC_MSR(APIC_LVR),		.always = false },
 115	{ .index = X2APIC_MSR(APIC_TASKPRI),		.always = false },
 116	{ .index = X2APIC_MSR(APIC_ARBPRI),		.always = false },
 117	{ .index = X2APIC_MSR(APIC_PROCPRI),		.always = false },
 118	{ .index = X2APIC_MSR(APIC_EOI),		.always = false },
 119	{ .index = X2APIC_MSR(APIC_RRR),		.always = false },
 120	{ .index = X2APIC_MSR(APIC_LDR),		.always = false },
 121	{ .index = X2APIC_MSR(APIC_DFR),		.always = false },
 122	{ .index = X2APIC_MSR(APIC_SPIV),		.always = false },
 123	{ .index = X2APIC_MSR(APIC_ISR),		.always = false },
 124	{ .index = X2APIC_MSR(APIC_TMR),		.always = false },
 125	{ .index = X2APIC_MSR(APIC_IRR),		.always = false },
 126	{ .index = X2APIC_MSR(APIC_ESR),		.always = false },
 127	{ .index = X2APIC_MSR(APIC_ICR),		.always = false },
 128	{ .index = X2APIC_MSR(APIC_ICR2),		.always = false },
 129
 130	/*
 131	 * Note:
 132	 * AMD does not virtualize APIC TSC-deadline timer mode, but it is
 133	 * emulated by KVM. When setting APIC LVTT (0x832) register bit 18,
 134	 * the AVIC hardware would generate GP fault. Therefore, always
 135	 * intercept the MSR 0x832, and do not setup direct_access_msr.
 136	 */
 137	{ .index = X2APIC_MSR(APIC_LVTTHMR),		.always = false },
 138	{ .index = X2APIC_MSR(APIC_LVTPC),		.always = false },
 139	{ .index = X2APIC_MSR(APIC_LVT0),		.always = false },
 140	{ .index = X2APIC_MSR(APIC_LVT1),		.always = false },
 141	{ .index = X2APIC_MSR(APIC_LVTERR),		.always = false },
 142	{ .index = X2APIC_MSR(APIC_TMICT),		.always = false },
 143	{ .index = X2APIC_MSR(APIC_TMCCT),		.always = false },
 144	{ .index = X2APIC_MSR(APIC_TDCR),		.always = false },
 145	{ .index = MSR_INVALID,				.always = false },
 146};
 147
 148/*
 149 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 150 * pause_filter_count: On processors that support Pause filtering(indicated
 151 *	by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
 152 *	count value. On VMRUN this value is loaded into an internal counter.
 153 *	Each time a pause instruction is executed, this counter is decremented
 154 *	until it reaches zero at which time a #VMEXIT is generated if pause
 155 *	intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
 156 *	Intercept Filtering for more details.
 157 *	This also indicate if ple logic enabled.
 158 *
 159 * pause_filter_thresh: In addition, some processor families support advanced
 160 *	pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
 161 *	the amount of time a guest is allowed to execute in a pause loop.
 162 *	In this mode, a 16-bit pause filter threshold field is added in the
 163 *	VMCB. The threshold value is a cycle count that is used to reset the
 164 *	pause counter. As with simple pause filtering, VMRUN loads the pause
 165 *	count value from VMCB into an internal counter. Then, on each pause
 166 *	instruction the hardware checks the elapsed number of cycles since
 167 *	the most recent pause instruction against the pause filter threshold.
 168 *	If the elapsed cycle count is greater than the pause filter threshold,
 169 *	then the internal pause count is reloaded from the VMCB and execution
 170 *	continues. If the elapsed cycle count is less than the pause filter
 171 *	threshold, then the internal pause count is decremented. If the count
 172 *	value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
 173 *	triggered. If advanced pause filtering is supported and pause filter
 174 *	threshold field is set to zero, the filter will operate in the simpler,
 175 *	count only mode.
 176 */
 177
 178static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
 179module_param(pause_filter_thresh, ushort, 0444);
 180
 181static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
 182module_param(pause_filter_count, ushort, 0444);
 183
 184/* Default doubles per-vcpu window every exit. */
 185static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 186module_param(pause_filter_count_grow, ushort, 0444);
 187
 188/* Default resets per-vcpu window every exit to pause_filter_count. */
 189static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 190module_param(pause_filter_count_shrink, ushort, 0444);
 191
 192/* Default is to compute the maximum so we can never overflow. */
 193static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
 194module_param(pause_filter_count_max, ushort, 0444);
 195
 196/*
 197 * Use nested page tables by default.  Note, NPT may get forced off by
 198 * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
 199 */
 200bool npt_enabled = true;
 201module_param_named(npt, npt_enabled, bool, 0444);
 202
 203/* allow nested virtualization in KVM/SVM */
 204static int nested = true;
 205module_param(nested, int, 0444);
 206
 207/* enable/disable Next RIP Save */
 208int nrips = true;
 209module_param(nrips, int, 0444);
 210
 211/* enable/disable Virtual VMLOAD VMSAVE */
 212static int vls = true;
 213module_param(vls, int, 0444);
 214
 215/* enable/disable Virtual GIF */
 216int vgif = true;
 217module_param(vgif, int, 0444);
 218
 219/* enable/disable LBR virtualization */
 220int lbrv = true;
 221module_param(lbrv, int, 0444);
 222
 223static int tsc_scaling = true;
 224module_param(tsc_scaling, int, 0444);
 225
 226/*
 227 * enable / disable AVIC.  Because the defaults differ for APICv
 228 * support between VMX and SVM we cannot use module_param_named.
 229 */
 230static bool avic;
 231module_param(avic, bool, 0444);
 232
 233bool __read_mostly dump_invalid_vmcb;
 234module_param(dump_invalid_vmcb, bool, 0644);
 235
 236
 237bool intercept_smi = true;
 238module_param(intercept_smi, bool, 0444);
 239
 240bool vnmi = true;
 241module_param(vnmi, bool, 0444);
 242
 243static bool svm_gp_erratum_intercept = true;
 244
 245static u8 rsm_ins_bytes[] = "\x0f\xaa";
 246
 247static unsigned long iopm_base;
 248
 249DEFINE_PER_CPU(struct svm_cpu_data, svm_data);
 250
 251/*
 252 * Only MSR_TSC_AUX is switched via the user return hook.  EFER is switched via
 253 * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
 254 *
 255 * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
 256 * defer the restoration of TSC_AUX until the CPU returns to userspace.
 257 */
 258static int tsc_aux_uret_slot __read_mostly = -1;
 259
 260static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 261
 262#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 263#define MSRS_RANGE_SIZE 2048
 264#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 265
 266u32 svm_msrpm_offset(u32 msr)
 267{
 268	u32 offset;
 269	int i;
 270
 271	for (i = 0; i < NUM_MSR_MAPS; i++) {
 272		if (msr < msrpm_ranges[i] ||
 273		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 274			continue;
 275
 276		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 277		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 278
 279		/* Now we have the u8 offset - but need the u32 offset */
 280		return offset / 4;
 281	}
 282
 283	/* MSR not in any range */
 284	return MSR_INVALID;
 285}
 286
 287static void svm_flush_tlb_current(struct kvm_vcpu *vcpu);
 288
 289static int get_npt_level(void)
 290{
 291#ifdef CONFIG_X86_64
 292	return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
 293#else
 294	return PT32E_ROOT_LEVEL;
 295#endif
 296}
 297
 298int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 299{
 300	struct vcpu_svm *svm = to_svm(vcpu);
 301	u64 old_efer = vcpu->arch.efer;
 302	vcpu->arch.efer = efer;
 303
 304	if (!npt_enabled) {
 305		/* Shadow paging assumes NX to be available.  */
 306		efer |= EFER_NX;
 307
 308		if (!(efer & EFER_LMA))
 309			efer &= ~EFER_LME;
 310	}
 311
 312	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
 313		if (!(efer & EFER_SVME)) {
 314			svm_leave_nested(vcpu);
 315			svm_set_gif(svm, true);
 316			/* #GP intercept is still needed for vmware backdoor */
 317			if (!enable_vmware_backdoor)
 318				clr_exception_intercept(svm, GP_VECTOR);
 319
 320			/*
 321			 * Free the nested guest state, unless we are in SMM.
 322			 * In this case we will return to the nested guest
 323			 * as soon as we leave SMM.
 324			 */
 325			if (!is_smm(vcpu))
 326				svm_free_nested(svm);
 327
 328		} else {
 329			int ret = svm_allocate_nested(svm);
 330
 331			if (ret) {
 332				vcpu->arch.efer = old_efer;
 333				return ret;
 334			}
 335
 336			/*
 337			 * Never intercept #GP for SEV guests, KVM can't
 338			 * decrypt guest memory to workaround the erratum.
 339			 */
 340			if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
 341				set_exception_intercept(svm, GP_VECTOR);
 342		}
 343	}
 344
 345	svm->vmcb->save.efer = efer | EFER_SVME;
 346	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
 347	return 0;
 348}
 349
 350static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 351{
 352	struct vcpu_svm *svm = to_svm(vcpu);
 353	u32 ret = 0;
 354
 355	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 356		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 357	return ret;
 358}
 359
 360static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 361{
 362	struct vcpu_svm *svm = to_svm(vcpu);
 363
 364	if (mask == 0)
 365		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 366	else
 367		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 368
 369}
 370
 371static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
 372					   bool commit_side_effects)
 373{
 374	struct vcpu_svm *svm = to_svm(vcpu);
 375	unsigned long old_rflags;
 376
 377	/*
 378	 * SEV-ES does not expose the next RIP. The RIP update is controlled by
 379	 * the type of exit and the #VC handler in the guest.
 380	 */
 381	if (sev_es_guest(vcpu->kvm))
 382		goto done;
 383
 384	if (nrips && svm->vmcb->control.next_rip != 0) {
 385		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 386		svm->next_rip = svm->vmcb->control.next_rip;
 387	}
 388
 389	if (!svm->next_rip) {
 390		if (unlikely(!commit_side_effects))
 391			old_rflags = svm->vmcb->save.rflags;
 392
 393		if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
 394			return 0;
 395
 396		if (unlikely(!commit_side_effects))
 397			svm->vmcb->save.rflags = old_rflags;
 398	} else {
 399		kvm_rip_write(vcpu, svm->next_rip);
 400	}
 401
 402done:
 403	if (likely(commit_side_effects))
 404		svm_set_interrupt_shadow(vcpu, 0);
 405
 406	return 1;
 407}
 408
 409static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
 410{
 411	return __svm_skip_emulated_instruction(vcpu, true);
 412}
 413
 414static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu)
 415{
 416	unsigned long rip, old_rip = kvm_rip_read(vcpu);
 417	struct vcpu_svm *svm = to_svm(vcpu);
 418
 419	/*
 420	 * Due to architectural shortcomings, the CPU doesn't always provide
 421	 * NextRIP, e.g. if KVM intercepted an exception that occurred while
 422	 * the CPU was vectoring an INTO/INT3 in the guest.  Temporarily skip
 423	 * the instruction even if NextRIP is supported to acquire the next
 424	 * RIP so that it can be shoved into the NextRIP field, otherwise
 425	 * hardware will fail to advance guest RIP during event injection.
 426	 * Drop the exception/interrupt if emulation fails and effectively
 427	 * retry the instruction, it's the least awful option.  If NRIPS is
 428	 * in use, the skip must not commit any side effects such as clearing
 429	 * the interrupt shadow or RFLAGS.RF.
 430	 */
 431	if (!__svm_skip_emulated_instruction(vcpu, !nrips))
 432		return -EIO;
 433
 434	rip = kvm_rip_read(vcpu);
 435
 436	/*
 437	 * Save the injection information, even when using next_rip, as the
 438	 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection
 439	 * doesn't complete due to a VM-Exit occurring while the CPU is
 440	 * vectoring the event.   Decoding the instruction isn't guaranteed to
 441	 * work as there may be no backing instruction, e.g. if the event is
 442	 * being injected by L1 for L2, or if the guest is patching INT3 into
 443	 * a different instruction.
 444	 */
 445	svm->soft_int_injected = true;
 446	svm->soft_int_csbase = svm->vmcb->save.cs.base;
 447	svm->soft_int_old_rip = old_rip;
 448	svm->soft_int_next_rip = rip;
 449
 450	if (nrips)
 451		kvm_rip_write(vcpu, old_rip);
 452
 453	if (static_cpu_has(X86_FEATURE_NRIPS))
 454		svm->vmcb->control.next_rip = rip;
 455
 456	return 0;
 457}
 458
 459static void svm_inject_exception(struct kvm_vcpu *vcpu)
 460{
 461	struct kvm_queued_exception *ex = &vcpu->arch.exception;
 462	struct vcpu_svm *svm = to_svm(vcpu);
 463
 464	kvm_deliver_exception_payload(vcpu, ex);
 465
 466	if (kvm_exception_is_soft(ex->vector) &&
 467	    svm_update_soft_interrupt_rip(vcpu))
 468		return;
 469
 470	svm->vmcb->control.event_inj = ex->vector
 471		| SVM_EVTINJ_VALID
 472		| (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 473		| SVM_EVTINJ_TYPE_EXEPT;
 474	svm->vmcb->control.event_inj_err = ex->error_code;
 475}
 476
 477static void svm_init_erratum_383(void)
 478{
 479	u32 low, high;
 480	int err;
 481	u64 val;
 482
 483	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 484		return;
 485
 486	/* Use _safe variants to not break nested virtualization */
 487	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 488	if (err)
 489		return;
 490
 491	val |= (1ULL << 47);
 492
 493	low  = lower_32_bits(val);
 494	high = upper_32_bits(val);
 495
 496	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 497
 498	erratum_383_found = true;
 499}
 500
 501static void svm_init_osvw(struct kvm_vcpu *vcpu)
 502{
 503	/*
 504	 * Guests should see errata 400 and 415 as fixed (assuming that
 505	 * HLT and IO instructions are intercepted).
 506	 */
 507	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 508	vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 509
 510	/*
 511	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
 512	 * all osvw.status bits inside that length, including bit 0 (which is
 513	 * reserved for erratum 298), are valid. However, if host processor's
 514	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
 515	 * be conservative here and therefore we tell the guest that erratum 298
 516	 * is present (because we really don't know).
 517	 */
 518	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 519		vcpu->arch.osvw.status |= 1;
 520}
 521
 522static bool __kvm_is_svm_supported(void)
 523{
 524	int cpu = smp_processor_id();
 525	struct cpuinfo_x86 *c = &cpu_data(cpu);
 526
 527	if (c->x86_vendor != X86_VENDOR_AMD &&
 528	    c->x86_vendor != X86_VENDOR_HYGON) {
 529		pr_err("CPU %d isn't AMD or Hygon\n", cpu);
 530		return false;
 531	}
 532
 533	if (!cpu_has(c, X86_FEATURE_SVM)) {
 534		pr_err("SVM not supported by CPU %d\n", cpu);
 535		return false;
 536	}
 537
 538	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
 539		pr_info("KVM is unsupported when running as an SEV guest\n");
 540		return false;
 541	}
 542
 543	return true;
 544}
 545
 546static bool kvm_is_svm_supported(void)
 547{
 548	bool supported;
 549
 550	migrate_disable();
 551	supported = __kvm_is_svm_supported();
 552	migrate_enable();
 553
 554	return supported;
 555}
 556
 557static int svm_check_processor_compat(void)
 558{
 559	if (!__kvm_is_svm_supported())
 560		return -EIO;
 561
 562	return 0;
 563}
 564
 565static void __svm_write_tsc_multiplier(u64 multiplier)
 566{
 567	if (multiplier == __this_cpu_read(current_tsc_ratio))
 568		return;
 569
 570	wrmsrl(MSR_AMD64_TSC_RATIO, multiplier);
 571	__this_cpu_write(current_tsc_ratio, multiplier);
 572}
 573
 574static __always_inline struct sev_es_save_area *sev_es_host_save_area(struct svm_cpu_data *sd)
 575{
 576	return &sd->save_area->host_sev_es_save;
 577}
 578
 579static inline void kvm_cpu_svm_disable(void)
 580{
 581	uint64_t efer;
 582
 583	wrmsrl(MSR_VM_HSAVE_PA, 0);
 584	rdmsrl(MSR_EFER, efer);
 585	if (efer & EFER_SVME) {
 586		/*
 587		 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
 588		 * NMI aren't blocked.
 589		 */
 590		stgi();
 591		wrmsrl(MSR_EFER, efer & ~EFER_SVME);
 592	}
 593}
 594
 595static void svm_emergency_disable_virtualization_cpu(void)
 596{
 597	kvm_rebooting = true;
 598
 599	kvm_cpu_svm_disable();
 600}
 601
 602static void svm_disable_virtualization_cpu(void)
 603{
 604	/* Make sure we clean up behind us */
 605	if (tsc_scaling)
 606		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
 607
 608	kvm_cpu_svm_disable();
 609
 610	amd_pmu_disable_virt();
 611}
 612
 613static int svm_enable_virtualization_cpu(void)
 614{
 615
 616	struct svm_cpu_data *sd;
 617	uint64_t efer;
 618	int me = raw_smp_processor_id();
 619
 620	rdmsrl(MSR_EFER, efer);
 621	if (efer & EFER_SVME)
 622		return -EBUSY;
 623
 624	sd = per_cpu_ptr(&svm_data, me);
 625	sd->asid_generation = 1;
 626	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 627	sd->next_asid = sd->max_asid + 1;
 628	sd->min_asid = max_sev_asid + 1;
 629
 630	wrmsrl(MSR_EFER, efer | EFER_SVME);
 631
 632	wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa);
 633
 634	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 635		/*
 636		 * Set the default value, even if we don't use TSC scaling
 637		 * to avoid having stale value in the msr
 638		 */
 639		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
 640	}
 641
 642
 643	/*
 644	 * Get OSVW bits.
 645	 *
 646	 * Note that it is possible to have a system with mixed processor
 647	 * revisions and therefore different OSVW bits. If bits are not the same
 648	 * on different processors then choose the worst case (i.e. if erratum
 649	 * is present on one processor and not on another then assume that the
 650	 * erratum is present everywhere).
 651	 */
 652	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 653		uint64_t len, status = 0;
 654		int err;
 655
 656		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 657		if (!err)
 658			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 659						      &err);
 660
 661		if (err)
 662			osvw_status = osvw_len = 0;
 663		else {
 664			if (len < osvw_len)
 665				osvw_len = len;
 666			osvw_status |= status;
 667			osvw_status &= (1ULL << osvw_len) - 1;
 668		}
 669	} else
 670		osvw_status = osvw_len = 0;
 671
 672	svm_init_erratum_383();
 673
 674	amd_pmu_enable_virt();
 675
 676	/*
 677	 * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type
 678	 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests.
 679	 * Since Linux does not change the value of TSC_AUX once set, prime the
 680	 * TSC_AUX field now to avoid a RDMSR on every vCPU run.
 681	 */
 682	if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
 
 683		u32 __maybe_unused msr_hi;
 684
 685		rdmsr(MSR_TSC_AUX, sev_es_host_save_area(sd)->tsc_aux, msr_hi);
 
 
 686	}
 687
 688	return 0;
 689}
 690
 691static void svm_cpu_uninit(int cpu)
 692{
 693	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
 694
 695	if (!sd->save_area)
 696		return;
 697
 698	kfree(sd->sev_vmcbs);
 699	__free_page(__sme_pa_to_page(sd->save_area_pa));
 700	sd->save_area_pa = 0;
 701	sd->save_area = NULL;
 702}
 703
 704static int svm_cpu_init(int cpu)
 705{
 706	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
 707	struct page *save_area_page;
 708	int ret = -ENOMEM;
 709
 710	memset(sd, 0, sizeof(struct svm_cpu_data));
 711	save_area_page = snp_safe_alloc_page_node(cpu_to_node(cpu), GFP_KERNEL);
 712	if (!save_area_page)
 713		return ret;
 714
 715	ret = sev_cpu_init(sd);
 716	if (ret)
 717		goto free_save_area;
 718
 719	sd->save_area = page_address(save_area_page);
 720	sd->save_area_pa = __sme_page_pa(save_area_page);
 721	return 0;
 722
 723free_save_area:
 724	__free_page(save_area_page);
 
 725	return ret;
 726
 727}
 728
 729static void set_dr_intercepts(struct vcpu_svm *svm)
 730{
 731	struct vmcb *vmcb = svm->vmcb01.ptr;
 732
 733	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ);
 734	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ);
 735	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ);
 736	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ);
 737	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ);
 738	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ);
 739	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ);
 740	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE);
 741	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE);
 742	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE);
 743	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE);
 744	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE);
 745	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE);
 746	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE);
 747	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ);
 748	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE);
 749
 750	recalc_intercepts(svm);
 751}
 752
 753static void clr_dr_intercepts(struct vcpu_svm *svm)
 754{
 755	struct vmcb *vmcb = svm->vmcb01.ptr;
 756
 757	vmcb->control.intercepts[INTERCEPT_DR] = 0;
 758
 759	recalc_intercepts(svm);
 760}
 761
 762static int direct_access_msr_slot(u32 msr)
 763{
 764	u32 i;
 765
 766	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 767		if (direct_access_msrs[i].index == msr)
 768			return i;
 769
 770	return -ENOENT;
 771}
 772
 773static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
 774				     int write)
 775{
 776	struct vcpu_svm *svm = to_svm(vcpu);
 777	int slot = direct_access_msr_slot(msr);
 778
 779	if (slot == -ENOENT)
 780		return;
 781
 782	/* Set the shadow bitmaps to the desired intercept states */
 783	if (read)
 784		set_bit(slot, svm->shadow_msr_intercept.read);
 785	else
 786		clear_bit(slot, svm->shadow_msr_intercept.read);
 787
 788	if (write)
 789		set_bit(slot, svm->shadow_msr_intercept.write);
 790	else
 791		clear_bit(slot, svm->shadow_msr_intercept.write);
 792}
 793
 794static bool valid_msr_intercept(u32 index)
 795{
 796	return direct_access_msr_slot(index) != -ENOENT;
 797}
 798
 799static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
 800{
 801	u8 bit_write;
 802	unsigned long tmp;
 803	u32 offset;
 804	u32 *msrpm;
 805
 806	/*
 807	 * For non-nested case:
 808	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
 809	 * save it.
 810	 *
 811	 * For nested case:
 812	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
 813	 * save it.
 814	 */
 815	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
 816				      to_svm(vcpu)->msrpm;
 817
 818	offset    = svm_msrpm_offset(msr);
 819	bit_write = 2 * (msr & 0x0f) + 1;
 820	tmp       = msrpm[offset];
 821
 822	BUG_ON(offset == MSR_INVALID);
 823
 824	return test_bit(bit_write, &tmp);
 825}
 826
 827static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
 828					u32 msr, int read, int write)
 829{
 830	struct vcpu_svm *svm = to_svm(vcpu);
 831	u8 bit_read, bit_write;
 832	unsigned long tmp;
 833	u32 offset;
 834
 835	/*
 836	 * If this warning triggers extend the direct_access_msrs list at the
 837	 * beginning of the file
 838	 */
 839	WARN_ON(!valid_msr_intercept(msr));
 840
 841	/* Enforce non allowed MSRs to trap */
 842	if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
 843		read = 0;
 844
 845	if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
 846		write = 0;
 847
 848	offset    = svm_msrpm_offset(msr);
 849	bit_read  = 2 * (msr & 0x0f);
 850	bit_write = 2 * (msr & 0x0f) + 1;
 851	tmp       = msrpm[offset];
 852
 853	BUG_ON(offset == MSR_INVALID);
 854
 855	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 856	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 857
 858	msrpm[offset] = tmp;
 859
 860	svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
 861	svm->nested.force_msr_bitmap_recalc = true;
 862}
 863
 864void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
 865			  int read, int write)
 866{
 867	set_shadow_msr_intercept(vcpu, msr, read, write);
 868	set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
 869}
 870
 871u32 *svm_vcpu_alloc_msrpm(void)
 872{
 873	unsigned int order = get_order(MSRPM_SIZE);
 874	struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
 875	u32 *msrpm;
 876
 877	if (!pages)
 878		return NULL;
 879
 880	msrpm = page_address(pages);
 881	memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
 882
 883	return msrpm;
 884}
 885
 886void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
 887{
 888	int i;
 889
 890	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 891		if (!direct_access_msrs[i].always)
 892			continue;
 893		set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
 894	}
 895}
 896
 897void svm_set_x2apic_msr_interception(struct vcpu_svm *svm, bool intercept)
 898{
 899	int i;
 900
 901	if (intercept == svm->x2avic_msrs_intercepted)
 902		return;
 903
 904	if (!x2avic_enabled)
 905		return;
 906
 907	for (i = 0; i < MAX_DIRECT_ACCESS_MSRS; i++) {
 908		int index = direct_access_msrs[i].index;
 909
 910		if ((index < APIC_BASE_MSR) ||
 911		    (index > APIC_BASE_MSR + 0xff))
 912			continue;
 913		set_msr_interception(&svm->vcpu, svm->msrpm, index,
 914				     !intercept, !intercept);
 915	}
 916
 917	svm->x2avic_msrs_intercepted = intercept;
 918}
 919
 920void svm_vcpu_free_msrpm(u32 *msrpm)
 921{
 922	__free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
 923}
 924
 925static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
 926{
 927	struct vcpu_svm *svm = to_svm(vcpu);
 928	u32 i;
 929
 930	/*
 931	 * Set intercept permissions for all direct access MSRs again. They
 932	 * will automatically get filtered through the MSR filter, so we are
 933	 * back in sync after this.
 934	 */
 935	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 936		u32 msr = direct_access_msrs[i].index;
 937		u32 read = test_bit(i, svm->shadow_msr_intercept.read);
 938		u32 write = test_bit(i, svm->shadow_msr_intercept.write);
 939
 940		set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
 941	}
 942}
 943
 944static void add_msr_offset(u32 offset)
 945{
 946	int i;
 947
 948	for (i = 0; i < MSRPM_OFFSETS; ++i) {
 949
 950		/* Offset already in list? */
 951		if (msrpm_offsets[i] == offset)
 952			return;
 953
 954		/* Slot used by another offset? */
 955		if (msrpm_offsets[i] != MSR_INVALID)
 956			continue;
 957
 958		/* Add offset to list */
 959		msrpm_offsets[i] = offset;
 960
 961		return;
 962	}
 963
 964	/*
 965	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
 966	 * increase MSRPM_OFFSETS in this case.
 967	 */
 968	BUG();
 969}
 970
 971static void init_msrpm_offsets(void)
 972{
 973	int i;
 974
 975	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
 976
 977	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 978		u32 offset;
 979
 980		offset = svm_msrpm_offset(direct_access_msrs[i].index);
 981		BUG_ON(offset == MSR_INVALID);
 982
 983		add_msr_offset(offset);
 984	}
 985}
 986
 987void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
 988{
 989	to_vmcb->save.dbgctl		= from_vmcb->save.dbgctl;
 990	to_vmcb->save.br_from		= from_vmcb->save.br_from;
 991	to_vmcb->save.br_to		= from_vmcb->save.br_to;
 992	to_vmcb->save.last_excp_from	= from_vmcb->save.last_excp_from;
 993	to_vmcb->save.last_excp_to	= from_vmcb->save.last_excp_to;
 994
 995	vmcb_mark_dirty(to_vmcb, VMCB_LBR);
 996}
 997
 998void svm_enable_lbrv(struct kvm_vcpu *vcpu)
 999{
1000	struct vcpu_svm *svm = to_svm(vcpu);
1001
1002	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1003	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1004	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1005	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1006	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1007
1008	if (sev_es_guest(vcpu->kvm))
1009		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_DEBUGCTLMSR, 1, 1);
1010
1011	/* Move the LBR msrs to the vmcb02 so that the guest can see them. */
1012	if (is_guest_mode(vcpu))
1013		svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr);
1014}
1015
1016static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
1017{
1018	struct vcpu_svm *svm = to_svm(vcpu);
1019
1020	KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm);
1021
1022	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1023	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1024	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1025	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1026	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1027
1028	/*
1029	 * Move the LBR msrs back to the vmcb01 to avoid copying them
1030	 * on nested guest entries.
1031	 */
1032	if (is_guest_mode(vcpu))
1033		svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb);
1034}
1035
1036static struct vmcb *svm_get_lbr_vmcb(struct vcpu_svm *svm)
1037{
1038	/*
1039	 * If LBR virtualization is disabled, the LBR MSRs are always kept in
1040	 * vmcb01.  If LBR virtualization is enabled and L1 is running VMs of
1041	 * its own, the MSRs are moved between vmcb01 and vmcb02 as needed.
1042	 */
1043	return svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK ? svm->vmcb :
1044								   svm->vmcb01.ptr;
1045}
1046
1047void svm_update_lbrv(struct kvm_vcpu *vcpu)
1048{
1049	struct vcpu_svm *svm = to_svm(vcpu);
1050	bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK;
1051	bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) ||
1052			    (is_guest_mode(vcpu) && guest_can_use(vcpu, X86_FEATURE_LBRV) &&
1053			    (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK));
1054
1055	if (enable_lbrv == current_enable_lbrv)
1056		return;
1057
1058	if (enable_lbrv)
1059		svm_enable_lbrv(vcpu);
1060	else
1061		svm_disable_lbrv(vcpu);
1062}
1063
1064void disable_nmi_singlestep(struct vcpu_svm *svm)
1065{
1066	svm->nmi_singlestep = false;
1067
1068	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1069		/* Clear our flags if they were not set by the guest */
1070		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1071			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1072		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1073			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1074	}
1075}
1076
1077static void grow_ple_window(struct kvm_vcpu *vcpu)
1078{
1079	struct vcpu_svm *svm = to_svm(vcpu);
1080	struct vmcb_control_area *control = &svm->vmcb->control;
1081	int old = control->pause_filter_count;
1082
1083	if (kvm_pause_in_guest(vcpu->kvm))
1084		return;
1085
1086	control->pause_filter_count = __grow_ple_window(old,
1087							pause_filter_count,
1088							pause_filter_count_grow,
1089							pause_filter_count_max);
1090
1091	if (control->pause_filter_count != old) {
1092		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1093		trace_kvm_ple_window_update(vcpu->vcpu_id,
1094					    control->pause_filter_count, old);
1095	}
1096}
1097
1098static void shrink_ple_window(struct kvm_vcpu *vcpu)
1099{
1100	struct vcpu_svm *svm = to_svm(vcpu);
1101	struct vmcb_control_area *control = &svm->vmcb->control;
1102	int old = control->pause_filter_count;
1103
1104	if (kvm_pause_in_guest(vcpu->kvm))
1105		return;
1106
1107	control->pause_filter_count =
1108				__shrink_ple_window(old,
1109						    pause_filter_count,
1110						    pause_filter_count_shrink,
1111						    pause_filter_count);
1112	if (control->pause_filter_count != old) {
1113		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1114		trace_kvm_ple_window_update(vcpu->vcpu_id,
1115					    control->pause_filter_count, old);
1116	}
1117}
1118
1119static void svm_hardware_unsetup(void)
1120{
1121	int cpu;
1122
1123	sev_hardware_unsetup();
1124
1125	for_each_possible_cpu(cpu)
1126		svm_cpu_uninit(cpu);
1127
1128	__free_pages(__sme_pa_to_page(iopm_base), get_order(IOPM_SIZE));
 
1129	iopm_base = 0;
1130}
1131
1132static void init_seg(struct vmcb_seg *seg)
1133{
1134	seg->selector = 0;
1135	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1136		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1137	seg->limit = 0xffff;
1138	seg->base = 0;
1139}
1140
1141static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1142{
1143	seg->selector = 0;
1144	seg->attrib = SVM_SELECTOR_P_MASK | type;
1145	seg->limit = 0xffff;
1146	seg->base = 0;
1147}
1148
1149static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
1150{
1151	struct vcpu_svm *svm = to_svm(vcpu);
1152
1153	return svm->nested.ctl.tsc_offset;
1154}
1155
1156static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
1157{
1158	struct vcpu_svm *svm = to_svm(vcpu);
1159
1160	return svm->tsc_ratio_msr;
1161}
1162
1163static void svm_write_tsc_offset(struct kvm_vcpu *vcpu)
1164{
1165	struct vcpu_svm *svm = to_svm(vcpu);
1166
1167	svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
1168	svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset;
1169	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1170}
1171
1172void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu)
1173{
1174	preempt_disable();
1175	if (to_svm(vcpu)->guest_state_loaded)
1176		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1177	preempt_enable();
1178}
1179
1180/* Evaluate instruction intercepts that depend on guest CPUID features. */
1181static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu,
1182					      struct vcpu_svm *svm)
1183{
1184	/*
1185	 * Intercept INVPCID if shadow paging is enabled to sync/free shadow
1186	 * roots, or if INVPCID is disabled in the guest to inject #UD.
1187	 */
1188	if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1189		if (!npt_enabled ||
1190		    !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID))
1191			svm_set_intercept(svm, INTERCEPT_INVPCID);
1192		else
1193			svm_clr_intercept(svm, INTERCEPT_INVPCID);
1194	}
1195
1196	if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
1197		if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1198			svm_clr_intercept(svm, INTERCEPT_RDTSCP);
1199		else
1200			svm_set_intercept(svm, INTERCEPT_RDTSCP);
1201	}
1202}
1203
1204static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu)
1205{
1206	struct vcpu_svm *svm = to_svm(vcpu);
1207
1208	if (guest_cpuid_is_intel_compatible(vcpu)) {
1209		/*
1210		 * We must intercept SYSENTER_EIP and SYSENTER_ESP
1211		 * accesses because the processor only stores 32 bits.
1212		 * For the same reason we cannot use virtual VMLOAD/VMSAVE.
1213		 */
1214		svm_set_intercept(svm, INTERCEPT_VMLOAD);
1215		svm_set_intercept(svm, INTERCEPT_VMSAVE);
1216		svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1217
1218		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0);
1219		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0);
1220	} else {
1221		/*
1222		 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1223		 * in VMCB and clear intercepts to avoid #VMEXIT.
1224		 */
1225		if (vls) {
1226			svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1227			svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1228			svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1229		}
1230		/* No need to intercept these MSRs */
1231		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
1232		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
1233	}
1234}
1235
1236static void init_vmcb(struct kvm_vcpu *vcpu)
1237{
1238	struct vcpu_svm *svm = to_svm(vcpu);
1239	struct vmcb *vmcb = svm->vmcb01.ptr;
1240	struct vmcb_control_area *control = &vmcb->control;
1241	struct vmcb_save_area *save = &vmcb->save;
1242
1243	svm_set_intercept(svm, INTERCEPT_CR0_READ);
1244	svm_set_intercept(svm, INTERCEPT_CR3_READ);
1245	svm_set_intercept(svm, INTERCEPT_CR4_READ);
1246	svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1247	svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1248	svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1249	if (!kvm_vcpu_apicv_active(vcpu))
1250		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1251
1252	set_dr_intercepts(svm);
1253
1254	set_exception_intercept(svm, PF_VECTOR);
1255	set_exception_intercept(svm, UD_VECTOR);
1256	set_exception_intercept(svm, MC_VECTOR);
1257	set_exception_intercept(svm, AC_VECTOR);
1258	set_exception_intercept(svm, DB_VECTOR);
1259	/*
1260	 * Guest access to VMware backdoor ports could legitimately
1261	 * trigger #GP because of TSS I/O permission bitmap.
1262	 * We intercept those #GP and allow access to them anyway
1263	 * as VMware does.
1264	 */
1265	if (enable_vmware_backdoor)
1266		set_exception_intercept(svm, GP_VECTOR);
1267
1268	svm_set_intercept(svm, INTERCEPT_INTR);
1269	svm_set_intercept(svm, INTERCEPT_NMI);
1270
1271	if (intercept_smi)
1272		svm_set_intercept(svm, INTERCEPT_SMI);
1273
1274	svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1275	svm_set_intercept(svm, INTERCEPT_RDPMC);
1276	svm_set_intercept(svm, INTERCEPT_CPUID);
1277	svm_set_intercept(svm, INTERCEPT_INVD);
1278	svm_set_intercept(svm, INTERCEPT_INVLPG);
1279	svm_set_intercept(svm, INTERCEPT_INVLPGA);
1280	svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1281	svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1282	svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1283	svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1284	svm_set_intercept(svm, INTERCEPT_VMRUN);
1285	svm_set_intercept(svm, INTERCEPT_VMMCALL);
1286	svm_set_intercept(svm, INTERCEPT_VMLOAD);
1287	svm_set_intercept(svm, INTERCEPT_VMSAVE);
1288	svm_set_intercept(svm, INTERCEPT_STGI);
1289	svm_set_intercept(svm, INTERCEPT_CLGI);
1290	svm_set_intercept(svm, INTERCEPT_SKINIT);
1291	svm_set_intercept(svm, INTERCEPT_WBINVD);
1292	svm_set_intercept(svm, INTERCEPT_XSETBV);
1293	svm_set_intercept(svm, INTERCEPT_RDPRU);
1294	svm_set_intercept(svm, INTERCEPT_RSM);
1295
1296	if (!kvm_mwait_in_guest(vcpu->kvm)) {
1297		svm_set_intercept(svm, INTERCEPT_MONITOR);
1298		svm_set_intercept(svm, INTERCEPT_MWAIT);
1299	}
1300
1301	if (!kvm_hlt_in_guest(vcpu->kvm))
1302		svm_set_intercept(svm, INTERCEPT_HLT);
1303
1304	control->iopm_base_pa = iopm_base;
1305	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1306	control->int_ctl = V_INTR_MASKING_MASK;
1307
1308	init_seg(&save->es);
1309	init_seg(&save->ss);
1310	init_seg(&save->ds);
1311	init_seg(&save->fs);
1312	init_seg(&save->gs);
1313
1314	save->cs.selector = 0xf000;
1315	save->cs.base = 0xffff0000;
1316	/* Executable/Readable Code Segment */
1317	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1318		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1319	save->cs.limit = 0xffff;
1320
1321	save->gdtr.base = 0;
1322	save->gdtr.limit = 0xffff;
1323	save->idtr.base = 0;
1324	save->idtr.limit = 0xffff;
1325
1326	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1327	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1328
1329	if (npt_enabled) {
1330		/* Setup VMCB for Nested Paging */
1331		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1332		svm_clr_intercept(svm, INTERCEPT_INVLPG);
1333		clr_exception_intercept(svm, PF_VECTOR);
1334		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1335		svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1336		save->g_pat = vcpu->arch.pat;
1337		save->cr3 = 0;
1338	}
1339	svm->current_vmcb->asid_generation = 0;
1340	svm->asid = 0;
1341
1342	svm->nested.vmcb12_gpa = INVALID_GPA;
1343	svm->nested.last_vmcb12_gpa = INVALID_GPA;
1344
1345	if (!kvm_pause_in_guest(vcpu->kvm)) {
1346		control->pause_filter_count = pause_filter_count;
1347		if (pause_filter_thresh)
1348			control->pause_filter_thresh = pause_filter_thresh;
1349		svm_set_intercept(svm, INTERCEPT_PAUSE);
1350	} else {
1351		svm_clr_intercept(svm, INTERCEPT_PAUSE);
1352	}
1353
1354	svm_recalc_instruction_intercepts(vcpu, svm);
1355
1356	/*
1357	 * If the host supports V_SPEC_CTRL then disable the interception
1358	 * of MSR_IA32_SPEC_CTRL.
1359	 */
1360	if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
1361		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
1362
1363	if (kvm_vcpu_apicv_active(vcpu))
1364		avic_init_vmcb(svm, vmcb);
1365
1366	if (vnmi)
1367		svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK;
1368
1369	if (vgif) {
1370		svm_clr_intercept(svm, INTERCEPT_STGI);
1371		svm_clr_intercept(svm, INTERCEPT_CLGI);
1372		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1373	}
1374
1375	if (sev_guest(vcpu->kvm))
1376		sev_init_vmcb(svm);
1377
1378	svm_hv_init_vmcb(vmcb);
1379	init_vmcb_after_set_cpuid(vcpu);
1380
1381	vmcb_mark_all_dirty(vmcb);
1382
1383	enable_gif(svm);
1384}
1385
1386static void __svm_vcpu_reset(struct kvm_vcpu *vcpu)
1387{
1388	struct vcpu_svm *svm = to_svm(vcpu);
1389
1390	svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1391
1392	svm_init_osvw(vcpu);
1393
1394	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS))
1395		vcpu->arch.microcode_version = 0x01000065;
1396	svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio;
1397
1398	svm->nmi_masked = false;
1399	svm->awaiting_iret_completion = false;
1400
1401	if (sev_es_guest(vcpu->kvm))
1402		sev_es_vcpu_reset(svm);
1403}
1404
1405static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1406{
1407	struct vcpu_svm *svm = to_svm(vcpu);
1408
1409	svm->spec_ctrl = 0;
1410	svm->virt_spec_ctrl = 0;
1411
1412	if (init_event)
1413		sev_snp_init_protected_guest_state(vcpu);
1414
1415	init_vmcb(vcpu);
1416
1417	if (!init_event)
1418		__svm_vcpu_reset(vcpu);
1419}
1420
1421void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1422{
1423	svm->current_vmcb = target_vmcb;
1424	svm->vmcb = target_vmcb->ptr;
1425}
1426
1427static int svm_vcpu_create(struct kvm_vcpu *vcpu)
1428{
1429	struct vcpu_svm *svm;
1430	struct page *vmcb01_page;
1431	struct page *vmsa_page = NULL;
1432	int err;
1433
1434	BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1435	svm = to_svm(vcpu);
1436
1437	err = -ENOMEM;
1438	vmcb01_page = snp_safe_alloc_page();
1439	if (!vmcb01_page)
1440		goto out;
1441
1442	if (sev_es_guest(vcpu->kvm)) {
1443		/*
1444		 * SEV-ES guests require a separate VMSA page used to contain
1445		 * the encrypted register state of the guest.
1446		 */
1447		vmsa_page = snp_safe_alloc_page();
1448		if (!vmsa_page)
1449			goto error_free_vmcb_page;
 
 
 
 
 
 
 
 
1450	}
1451
1452	err = avic_init_vcpu(svm);
1453	if (err)
1454		goto error_free_vmsa_page;
1455
1456	svm->msrpm = svm_vcpu_alloc_msrpm();
1457	if (!svm->msrpm) {
1458		err = -ENOMEM;
1459		goto error_free_vmsa_page;
1460	}
1461
1462	svm->x2avic_msrs_intercepted = true;
1463
1464	svm->vmcb01.ptr = page_address(vmcb01_page);
1465	svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1466	svm_switch_vmcb(svm, &svm->vmcb01);
1467
1468	if (vmsa_page)
1469		svm->sev_es.vmsa = page_address(vmsa_page);
1470
1471	svm->guest_state_loaded = false;
1472
1473	return 0;
1474
1475error_free_vmsa_page:
1476	if (vmsa_page)
1477		__free_page(vmsa_page);
1478error_free_vmcb_page:
1479	__free_page(vmcb01_page);
1480out:
1481	return err;
1482}
1483
1484static void svm_clear_current_vmcb(struct vmcb *vmcb)
1485{
1486	int i;
1487
1488	for_each_online_cpu(i)
1489		cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL);
1490}
1491
1492static void svm_vcpu_free(struct kvm_vcpu *vcpu)
1493{
1494	struct vcpu_svm *svm = to_svm(vcpu);
1495
1496	/*
1497	 * The vmcb page can be recycled, causing a false negative in
1498	 * svm_vcpu_load(). So, ensure that no logical CPU has this
1499	 * vmcb page recorded as its current vmcb.
1500	 */
1501	svm_clear_current_vmcb(svm->vmcb);
1502
1503	svm_leave_nested(vcpu);
1504	svm_free_nested(svm);
1505
1506	sev_free_vcpu(vcpu);
1507
1508	__free_page(__sme_pa_to_page(svm->vmcb01.pa));
1509	__free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1510}
1511
1512static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1513{
1514	struct vcpu_svm *svm = to_svm(vcpu);
1515	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
1516
1517	if (sev_es_guest(vcpu->kvm))
1518		sev_es_unmap_ghcb(svm);
1519
1520	if (svm->guest_state_loaded)
1521		return;
1522
1523	/*
1524	 * Save additional host state that will be restored on VMEXIT (sev-es)
1525	 * or subsequent vmload of host save area.
1526	 */
1527	vmsave(sd->save_area_pa);
1528	if (sev_es_guest(vcpu->kvm))
1529		sev_es_prepare_switch_to_guest(svm, sev_es_host_save_area(sd));
 
 
 
 
1530
1531	if (tsc_scaling)
1532		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1533
1534	/*
1535	 * TSC_AUX is always virtualized for SEV-ES guests when the feature is
1536	 * available. The user return MSR support is not required in this case
1537	 * because TSC_AUX is restored on #VMEXIT from the host save area
1538	 * (which has been initialized in svm_enable_virtualization_cpu()).
1539	 */
1540	if (likely(tsc_aux_uret_slot >= 0) &&
1541	    (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))
1542		kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1543
1544	svm->guest_state_loaded = true;
1545}
1546
1547static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1548{
1549	to_svm(vcpu)->guest_state_loaded = false;
1550}
1551
1552static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1553{
1554	struct vcpu_svm *svm = to_svm(vcpu);
1555	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
1556
1557	if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm))
1558		shrink_ple_window(vcpu);
1559
1560	if (sd->current_vmcb != svm->vmcb) {
1561		sd->current_vmcb = svm->vmcb;
1562
1563		if (!cpu_feature_enabled(X86_FEATURE_IBPB_ON_VMEXIT))
1564			indirect_branch_prediction_barrier();
1565	}
1566	if (kvm_vcpu_apicv_active(vcpu))
1567		avic_vcpu_load(vcpu, cpu);
1568}
1569
1570static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1571{
1572	if (kvm_vcpu_apicv_active(vcpu))
1573		avic_vcpu_put(vcpu);
1574
1575	svm_prepare_host_switch(vcpu);
1576
1577	++vcpu->stat.host_state_reload;
1578}
1579
1580static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1581{
1582	struct vcpu_svm *svm = to_svm(vcpu);
1583	unsigned long rflags = svm->vmcb->save.rflags;
1584
1585	if (svm->nmi_singlestep) {
1586		/* Hide our flags if they were not set by the guest */
1587		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1588			rflags &= ~X86_EFLAGS_TF;
1589		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1590			rflags &= ~X86_EFLAGS_RF;
1591	}
1592	return rflags;
1593}
1594
1595static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1596{
1597	if (to_svm(vcpu)->nmi_singlestep)
1598		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1599
1600       /*
1601        * Any change of EFLAGS.VM is accompanied by a reload of SS
1602        * (caused by either a task switch or an inter-privilege IRET),
1603        * so we do not need to update the CPL here.
1604        */
1605	to_svm(vcpu)->vmcb->save.rflags = rflags;
1606}
1607
1608static bool svm_get_if_flag(struct kvm_vcpu *vcpu)
1609{
1610	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
1611
1612	return sev_es_guest(vcpu->kvm)
1613		? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK
1614		: kvm_get_rflags(vcpu) & X86_EFLAGS_IF;
1615}
1616
1617static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1618{
1619	kvm_register_mark_available(vcpu, reg);
1620
1621	switch (reg) {
1622	case VCPU_EXREG_PDPTR:
1623		/*
1624		 * When !npt_enabled, mmu->pdptrs[] is already available since
1625		 * it is always updated per SDM when moving to CRs.
1626		 */
1627		if (npt_enabled)
1628			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
1629		break;
1630	default:
1631		KVM_BUG_ON(1, vcpu->kvm);
1632	}
1633}
1634
1635static void svm_set_vintr(struct vcpu_svm *svm)
1636{
1637	struct vmcb_control_area *control;
1638
1639	/*
1640	 * The following fields are ignored when AVIC is enabled
1641	 */
1642	WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu));
1643
1644	svm_set_intercept(svm, INTERCEPT_VINTR);
1645
1646	/*
1647	 * Recalculating intercepts may have cleared the VINTR intercept.  If
1648	 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF
1649	 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN.
1650	 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as
1651	 * interrupts will never be unblocked while L2 is running.
1652	 */
1653	if (!svm_is_intercept(svm, INTERCEPT_VINTR))
1654		return;
1655
1656	/*
1657	 * This is just a dummy VINTR to actually cause a vmexit to happen.
1658	 * Actual injection of virtual interrupts happens through EVENTINJ.
1659	 */
1660	control = &svm->vmcb->control;
1661	control->int_vector = 0x0;
1662	control->int_ctl &= ~V_INTR_PRIO_MASK;
1663	control->int_ctl |= V_IRQ_MASK |
1664		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1665	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1666}
1667
1668static void svm_clear_vintr(struct vcpu_svm *svm)
1669{
1670	svm_clr_intercept(svm, INTERCEPT_VINTR);
1671
1672	/* Drop int_ctl fields related to VINTR injection.  */
1673	svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1674	if (is_guest_mode(&svm->vcpu)) {
1675		svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1676
1677		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1678			(svm->nested.ctl.int_ctl & V_TPR_MASK));
1679
1680		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1681			V_IRQ_INJECTION_BITS_MASK;
1682
1683		svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1684	}
1685
1686	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1687}
1688
1689static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1690{
1691	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1692	struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1693
1694	switch (seg) {
1695	case VCPU_SREG_CS: return &save->cs;
1696	case VCPU_SREG_DS: return &save->ds;
1697	case VCPU_SREG_ES: return &save->es;
1698	case VCPU_SREG_FS: return &save01->fs;
1699	case VCPU_SREG_GS: return &save01->gs;
1700	case VCPU_SREG_SS: return &save->ss;
1701	case VCPU_SREG_TR: return &save01->tr;
1702	case VCPU_SREG_LDTR: return &save01->ldtr;
1703	}
1704	BUG();
1705	return NULL;
1706}
1707
1708static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1709{
1710	struct vmcb_seg *s = svm_seg(vcpu, seg);
1711
1712	return s->base;
1713}
1714
1715static void svm_get_segment(struct kvm_vcpu *vcpu,
1716			    struct kvm_segment *var, int seg)
1717{
1718	struct vmcb_seg *s = svm_seg(vcpu, seg);
1719
1720	var->base = s->base;
1721	var->limit = s->limit;
1722	var->selector = s->selector;
1723	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1724	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1725	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1726	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1727	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1728	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1729	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1730
1731	/*
1732	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1733	 * However, the SVM spec states that the G bit is not observed by the
1734	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1735	 * So let's synthesize a legal G bit for all segments, this helps
1736	 * running KVM nested. It also helps cross-vendor migration, because
1737	 * Intel's vmentry has a check on the 'G' bit.
1738	 */
1739	var->g = s->limit > 0xfffff;
1740
1741	/*
1742	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1743	 * for cross vendor migration purposes by "not present"
1744	 */
1745	var->unusable = !var->present;
1746
1747	switch (seg) {
1748	case VCPU_SREG_TR:
1749		/*
1750		 * Work around a bug where the busy flag in the tr selector
1751		 * isn't exposed
1752		 */
1753		var->type |= 0x2;
1754		break;
1755	case VCPU_SREG_DS:
1756	case VCPU_SREG_ES:
1757	case VCPU_SREG_FS:
1758	case VCPU_SREG_GS:
1759		/*
1760		 * The accessed bit must always be set in the segment
1761		 * descriptor cache, although it can be cleared in the
1762		 * descriptor, the cached bit always remains at 1. Since
1763		 * Intel has a check on this, set it here to support
1764		 * cross-vendor migration.
1765		 */
1766		if (!var->unusable)
1767			var->type |= 0x1;
1768		break;
1769	case VCPU_SREG_SS:
1770		/*
1771		 * On AMD CPUs sometimes the DB bit in the segment
1772		 * descriptor is left as 1, although the whole segment has
1773		 * been made unusable. Clear it here to pass an Intel VMX
1774		 * entry check when cross vendor migrating.
1775		 */
1776		if (var->unusable)
1777			var->db = 0;
1778		/* This is symmetric with svm_set_segment() */
1779		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1780		break;
1781	}
1782}
1783
1784static int svm_get_cpl(struct kvm_vcpu *vcpu)
1785{
1786	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1787
1788	return save->cpl;
1789}
1790
1791static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1792{
1793	struct kvm_segment cs;
1794
1795	svm_get_segment(vcpu, &cs, VCPU_SREG_CS);
1796	*db = cs.db;
1797	*l = cs.l;
1798}
1799
1800static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1801{
1802	struct vcpu_svm *svm = to_svm(vcpu);
1803
1804	dt->size = svm->vmcb->save.idtr.limit;
1805	dt->address = svm->vmcb->save.idtr.base;
1806}
1807
1808static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1809{
1810	struct vcpu_svm *svm = to_svm(vcpu);
1811
1812	svm->vmcb->save.idtr.limit = dt->size;
1813	svm->vmcb->save.idtr.base = dt->address ;
1814	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1815}
1816
1817static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1818{
1819	struct vcpu_svm *svm = to_svm(vcpu);
1820
1821	dt->size = svm->vmcb->save.gdtr.limit;
1822	dt->address = svm->vmcb->save.gdtr.base;
1823}
1824
1825static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1826{
1827	struct vcpu_svm *svm = to_svm(vcpu);
1828
1829	svm->vmcb->save.gdtr.limit = dt->size;
1830	svm->vmcb->save.gdtr.base = dt->address ;
1831	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1832}
1833
1834static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1835{
1836	struct vcpu_svm *svm = to_svm(vcpu);
1837
1838	/*
1839	 * For guests that don't set guest_state_protected, the cr3 update is
1840	 * handled via kvm_mmu_load() while entering the guest. For guests
1841	 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to
1842	 * VMCB save area now, since the save area will become the initial
1843	 * contents of the VMSA, and future VMCB save area updates won't be
1844	 * seen.
1845	 */
1846	if (sev_es_guest(vcpu->kvm)) {
1847		svm->vmcb->save.cr3 = cr3;
1848		vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1849	}
1850}
1851
1852static bool svm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1853{
1854	return true;
1855}
1856
1857void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1858{
1859	struct vcpu_svm *svm = to_svm(vcpu);
1860	u64 hcr0 = cr0;
1861	bool old_paging = is_paging(vcpu);
1862
1863#ifdef CONFIG_X86_64
1864	if (vcpu->arch.efer & EFER_LME) {
1865		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1866			vcpu->arch.efer |= EFER_LMA;
1867			if (!vcpu->arch.guest_state_protected)
1868				svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1869		}
1870
1871		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1872			vcpu->arch.efer &= ~EFER_LMA;
1873			if (!vcpu->arch.guest_state_protected)
1874				svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1875		}
1876	}
1877#endif
1878	vcpu->arch.cr0 = cr0;
1879
1880	if (!npt_enabled) {
1881		hcr0 |= X86_CR0_PG | X86_CR0_WP;
1882		if (old_paging != is_paging(vcpu))
1883			svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
1884	}
1885
1886	/*
1887	 * re-enable caching here because the QEMU bios
1888	 * does not do it - this results in some delay at
1889	 * reboot
1890	 */
1891	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1892		hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1893
1894	svm->vmcb->save.cr0 = hcr0;
1895	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1896
1897	/*
1898	 * SEV-ES guests must always keep the CR intercepts cleared. CR
1899	 * tracking is done using the CR write traps.
1900	 */
1901	if (sev_es_guest(vcpu->kvm))
1902		return;
1903
1904	if (hcr0 == cr0) {
1905		/* Selective CR0 write remains on.  */
1906		svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1907		svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1908	} else {
1909		svm_set_intercept(svm, INTERCEPT_CR0_READ);
1910		svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1911	}
1912}
1913
1914static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1915{
1916	return true;
1917}
1918
1919void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1920{
1921	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1922	unsigned long old_cr4 = vcpu->arch.cr4;
1923
1924	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1925		svm_flush_tlb_current(vcpu);
1926
1927	vcpu->arch.cr4 = cr4;
1928	if (!npt_enabled) {
1929		cr4 |= X86_CR4_PAE;
1930
1931		if (!is_paging(vcpu))
1932			cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
1933	}
1934	cr4 |= host_cr4_mce;
1935	to_svm(vcpu)->vmcb->save.cr4 = cr4;
1936	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1937
1938	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1939		kvm_update_cpuid_runtime(vcpu);
1940}
1941
1942static void svm_set_segment(struct kvm_vcpu *vcpu,
1943			    struct kvm_segment *var, int seg)
1944{
1945	struct vcpu_svm *svm = to_svm(vcpu);
1946	struct vmcb_seg *s = svm_seg(vcpu, seg);
1947
1948	s->base = var->base;
1949	s->limit = var->limit;
1950	s->selector = var->selector;
1951	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1952	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1953	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1954	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1955	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1956	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1957	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1958	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1959
1960	/*
1961	 * This is always accurate, except if SYSRET returned to a segment
1962	 * with SS.DPL != 3.  Intel does not have this quirk, and always
1963	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1964	 * would entail passing the CPL to userspace and back.
1965	 */
1966	if (seg == VCPU_SREG_SS)
1967		/* This is symmetric with svm_get_segment() */
1968		svm->vmcb->save.cpl = (var->dpl & 3);
1969
1970	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1971}
1972
1973static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1974{
1975	struct vcpu_svm *svm = to_svm(vcpu);
1976
1977	clr_exception_intercept(svm, BP_VECTOR);
1978
1979	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1980		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1981			set_exception_intercept(svm, BP_VECTOR);
1982	}
1983}
1984
1985static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1986{
1987	if (sd->next_asid > sd->max_asid) {
1988		++sd->asid_generation;
1989		sd->next_asid = sd->min_asid;
1990		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1991		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1992	}
1993
1994	svm->current_vmcb->asid_generation = sd->asid_generation;
1995	svm->asid = sd->next_asid++;
1996}
1997
1998static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
1999{
2000	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
2001
2002	if (vcpu->arch.guest_state_protected)
2003		return;
2004
2005	if (unlikely(value != vmcb->save.dr6)) {
2006		vmcb->save.dr6 = value;
2007		vmcb_mark_dirty(vmcb, VMCB_DR);
2008	}
2009}
2010
2011static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2012{
2013	struct vcpu_svm *svm = to_svm(vcpu);
2014
2015	if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm)))
2016		return;
2017
2018	get_debugreg(vcpu->arch.db[0], 0);
2019	get_debugreg(vcpu->arch.db[1], 1);
2020	get_debugreg(vcpu->arch.db[2], 2);
2021	get_debugreg(vcpu->arch.db[3], 3);
2022	/*
2023	 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
2024	 * because db_interception might need it.  We can do it before vmentry.
2025	 */
2026	vcpu->arch.dr6 = svm->vmcb->save.dr6;
2027	vcpu->arch.dr7 = svm->vmcb->save.dr7;
2028	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2029	set_dr_intercepts(svm);
2030}
2031
2032static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2033{
2034	struct vcpu_svm *svm = to_svm(vcpu);
2035
2036	if (vcpu->arch.guest_state_protected)
2037		return;
2038
2039	svm->vmcb->save.dr7 = value;
2040	vmcb_mark_dirty(svm->vmcb, VMCB_DR);
2041}
2042
2043static int pf_interception(struct kvm_vcpu *vcpu)
2044{
2045	struct vcpu_svm *svm = to_svm(vcpu);
2046
2047	u64 fault_address = svm->vmcb->control.exit_info_2;
2048	u64 error_code = svm->vmcb->control.exit_info_1;
2049
2050	return kvm_handle_page_fault(vcpu, error_code, fault_address,
2051			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2052			svm->vmcb->control.insn_bytes : NULL,
2053			svm->vmcb->control.insn_len);
2054}
2055
2056static int npf_interception(struct kvm_vcpu *vcpu)
2057{
2058	struct vcpu_svm *svm = to_svm(vcpu);
2059	int rc;
2060
2061	u64 fault_address = svm->vmcb->control.exit_info_2;
2062	u64 error_code = svm->vmcb->control.exit_info_1;
2063
2064	/*
2065	 * WARN if hardware generates a fault with an error code that collides
2066	 * with KVM-defined sythentic flags.  Clear the flags and continue on,
2067	 * i.e. don't terminate the VM, as KVM can't possibly be relying on a
2068	 * flag that KVM doesn't know about.
2069	 */
2070	if (WARN_ON_ONCE(error_code & PFERR_SYNTHETIC_MASK))
2071		error_code &= ~PFERR_SYNTHETIC_MASK;
2072
2073	if (sev_snp_guest(vcpu->kvm) && (error_code & PFERR_GUEST_ENC_MASK))
2074		error_code |= PFERR_PRIVATE_ACCESS;
2075
2076	trace_kvm_page_fault(vcpu, fault_address, error_code);
2077	rc = kvm_mmu_page_fault(vcpu, fault_address, error_code,
2078				static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2079				svm->vmcb->control.insn_bytes : NULL,
2080				svm->vmcb->control.insn_len);
2081
2082	if (rc > 0 && error_code & PFERR_GUEST_RMP_MASK)
2083		sev_handle_rmp_fault(vcpu, fault_address, error_code);
2084
2085	return rc;
2086}
2087
2088static int db_interception(struct kvm_vcpu *vcpu)
2089{
2090	struct kvm_run *kvm_run = vcpu->run;
2091	struct vcpu_svm *svm = to_svm(vcpu);
2092
2093	if (!(vcpu->guest_debug &
2094	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2095		!svm->nmi_singlestep) {
2096		u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
2097		kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
2098		return 1;
2099	}
2100
2101	if (svm->nmi_singlestep) {
2102		disable_nmi_singlestep(svm);
2103		/* Make sure we check for pending NMIs upon entry */
2104		kvm_make_request(KVM_REQ_EVENT, vcpu);
2105	}
2106
2107	if (vcpu->guest_debug &
2108	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2109		kvm_run->exit_reason = KVM_EXIT_DEBUG;
2110		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
2111		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
2112		kvm_run->debug.arch.pc =
2113			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2114		kvm_run->debug.arch.exception = DB_VECTOR;
2115		return 0;
2116	}
2117
2118	return 1;
2119}
2120
2121static int bp_interception(struct kvm_vcpu *vcpu)
2122{
2123	struct vcpu_svm *svm = to_svm(vcpu);
2124	struct kvm_run *kvm_run = vcpu->run;
2125
2126	kvm_run->exit_reason = KVM_EXIT_DEBUG;
2127	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2128	kvm_run->debug.arch.exception = BP_VECTOR;
2129	return 0;
2130}
2131
2132static int ud_interception(struct kvm_vcpu *vcpu)
2133{
2134	return handle_ud(vcpu);
2135}
2136
2137static int ac_interception(struct kvm_vcpu *vcpu)
2138{
2139	kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
2140	return 1;
2141}
2142
2143static bool is_erratum_383(void)
2144{
2145	int err, i;
2146	u64 value;
2147
2148	if (!erratum_383_found)
2149		return false;
2150
2151	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2152	if (err)
2153		return false;
2154
2155	/* Bit 62 may or may not be set for this mce */
2156	value &= ~(1ULL << 62);
2157
2158	if (value != 0xb600000000010015ULL)
2159		return false;
2160
2161	/* Clear MCi_STATUS registers */
2162	for (i = 0; i < 6; ++i)
2163		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2164
2165	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2166	if (!err) {
2167		u32 low, high;
2168
2169		value &= ~(1ULL << 2);
2170		low    = lower_32_bits(value);
2171		high   = upper_32_bits(value);
2172
2173		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2174	}
2175
2176	/* Flush tlb to evict multi-match entries */
2177	__flush_tlb_all();
2178
2179	return true;
2180}
2181
2182static void svm_handle_mce(struct kvm_vcpu *vcpu)
2183{
2184	if (is_erratum_383()) {
2185		/*
2186		 * Erratum 383 triggered. Guest state is corrupt so kill the
2187		 * guest.
2188		 */
2189		pr_err("Guest triggered AMD Erratum 383\n");
2190
2191		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2192
2193		return;
2194	}
2195
2196	/*
2197	 * On an #MC intercept the MCE handler is not called automatically in
2198	 * the host. So do it by hand here.
2199	 */
2200	kvm_machine_check();
2201}
2202
2203static int mc_interception(struct kvm_vcpu *vcpu)
2204{
2205	return 1;
2206}
2207
2208static int shutdown_interception(struct kvm_vcpu *vcpu)
2209{
2210	struct kvm_run *kvm_run = vcpu->run;
2211	struct vcpu_svm *svm = to_svm(vcpu);
2212
2213
2214	/*
2215	 * VMCB is undefined after a SHUTDOWN intercept.  INIT the vCPU to put
2216	 * the VMCB in a known good state.  Unfortuately, KVM doesn't have
2217	 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
2218	 * userspace.  At a platform view, INIT is acceptable behavior as
2219	 * there exist bare metal platforms that automatically INIT the CPU
2220	 * in response to shutdown.
2221	 *
2222	 * The VM save area for SEV-ES guests has already been encrypted so it
2223	 * cannot be reinitialized, i.e. synthesizing INIT is futile.
2224	 */
2225	if (!sev_es_guest(vcpu->kvm)) {
2226		clear_page(svm->vmcb);
2227		kvm_vcpu_reset(vcpu, true);
2228	}
2229
2230	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2231	return 0;
2232}
2233
2234static int io_interception(struct kvm_vcpu *vcpu)
2235{
2236	struct vcpu_svm *svm = to_svm(vcpu);
2237	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2238	int size, in, string;
2239	unsigned port;
2240
2241	++vcpu->stat.io_exits;
2242	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2243	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2244	port = io_info >> 16;
2245	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2246
2247	if (string) {
2248		if (sev_es_guest(vcpu->kvm))
2249			return sev_es_string_io(svm, size, port, in);
2250		else
2251			return kvm_emulate_instruction(vcpu, 0);
2252	}
2253
2254	svm->next_rip = svm->vmcb->control.exit_info_2;
2255
2256	return kvm_fast_pio(vcpu, size, port, in);
2257}
2258
2259static int nmi_interception(struct kvm_vcpu *vcpu)
2260{
2261	return 1;
2262}
2263
2264static int smi_interception(struct kvm_vcpu *vcpu)
2265{
2266	return 1;
2267}
2268
2269static int intr_interception(struct kvm_vcpu *vcpu)
2270{
2271	++vcpu->stat.irq_exits;
2272	return 1;
2273}
2274
2275static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2276{
2277	struct vcpu_svm *svm = to_svm(vcpu);
2278	struct vmcb *vmcb12;
2279	struct kvm_host_map map;
2280	int ret;
2281
2282	if (nested_svm_check_permissions(vcpu))
2283		return 1;
2284
2285	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2286	if (ret) {
2287		if (ret == -EINVAL)
2288			kvm_inject_gp(vcpu, 0);
2289		return 1;
2290	}
2291
2292	vmcb12 = map.hva;
2293
2294	ret = kvm_skip_emulated_instruction(vcpu);
2295
2296	if (vmload) {
2297		svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
2298		svm->sysenter_eip_hi = 0;
2299		svm->sysenter_esp_hi = 0;
2300	} else {
2301		svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
2302	}
2303
2304	kvm_vcpu_unmap(vcpu, &map);
2305
2306	return ret;
2307}
2308
2309static int vmload_interception(struct kvm_vcpu *vcpu)
2310{
2311	return vmload_vmsave_interception(vcpu, true);
2312}
2313
2314static int vmsave_interception(struct kvm_vcpu *vcpu)
2315{
2316	return vmload_vmsave_interception(vcpu, false);
2317}
2318
2319static int vmrun_interception(struct kvm_vcpu *vcpu)
2320{
2321	if (nested_svm_check_permissions(vcpu))
2322		return 1;
2323
2324	return nested_svm_vmrun(vcpu);
2325}
2326
2327enum {
2328	NONE_SVM_INSTR,
2329	SVM_INSTR_VMRUN,
2330	SVM_INSTR_VMLOAD,
2331	SVM_INSTR_VMSAVE,
2332};
2333
2334/* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
2335static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2336{
2337	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2338
2339	if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2340		return NONE_SVM_INSTR;
2341
2342	switch (ctxt->modrm) {
2343	case 0xd8: /* VMRUN */
2344		return SVM_INSTR_VMRUN;
2345	case 0xda: /* VMLOAD */
2346		return SVM_INSTR_VMLOAD;
2347	case 0xdb: /* VMSAVE */
2348		return SVM_INSTR_VMSAVE;
2349	default:
2350		break;
2351	}
2352
2353	return NONE_SVM_INSTR;
2354}
2355
2356static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2357{
2358	const int guest_mode_exit_codes[] = {
2359		[SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2360		[SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2361		[SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2362	};
2363	int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2364		[SVM_INSTR_VMRUN] = vmrun_interception,
2365		[SVM_INSTR_VMLOAD] = vmload_interception,
2366		[SVM_INSTR_VMSAVE] = vmsave_interception,
2367	};
2368	struct vcpu_svm *svm = to_svm(vcpu);
2369	int ret;
2370
2371	if (is_guest_mode(vcpu)) {
2372		/* Returns '1' or -errno on failure, '0' on success. */
2373		ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2374		if (ret)
2375			return ret;
2376		return 1;
2377	}
2378	return svm_instr_handlers[opcode](vcpu);
2379}
2380
2381/*
2382 * #GP handling code. Note that #GP can be triggered under the following two
2383 * cases:
2384 *   1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2385 *      some AMD CPUs when EAX of these instructions are in the reserved memory
2386 *      regions (e.g. SMM memory on host).
2387 *   2) VMware backdoor
2388 */
2389static int gp_interception(struct kvm_vcpu *vcpu)
2390{
2391	struct vcpu_svm *svm = to_svm(vcpu);
2392	u32 error_code = svm->vmcb->control.exit_info_1;
2393	int opcode;
2394
2395	/* Both #GP cases have zero error_code */
2396	if (error_code)
2397		goto reinject;
2398
2399	/* Decode the instruction for usage later */
2400	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2401		goto reinject;
2402
2403	opcode = svm_instr_opcode(vcpu);
2404
2405	if (opcode == NONE_SVM_INSTR) {
2406		if (!enable_vmware_backdoor)
2407			goto reinject;
2408
2409		/*
2410		 * VMware backdoor emulation on #GP interception only handles
2411		 * IN{S}, OUT{S}, and RDPMC.
2412		 */
2413		if (!is_guest_mode(vcpu))
2414			return kvm_emulate_instruction(vcpu,
2415				EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2416	} else {
2417		/* All SVM instructions expect page aligned RAX */
2418		if (svm->vmcb->save.rax & ~PAGE_MASK)
2419			goto reinject;
2420
2421		return emulate_svm_instr(vcpu, opcode);
2422	}
2423
2424reinject:
2425	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2426	return 1;
2427}
2428
2429void svm_set_gif(struct vcpu_svm *svm, bool value)
2430{
2431	if (value) {
2432		/*
2433		 * If VGIF is enabled, the STGI intercept is only added to
2434		 * detect the opening of the SMI/NMI window; remove it now.
2435		 * Likewise, clear the VINTR intercept, we will set it
2436		 * again while processing KVM_REQ_EVENT if needed.
2437		 */
2438		if (vgif)
2439			svm_clr_intercept(svm, INTERCEPT_STGI);
2440		if (svm_is_intercept(svm, INTERCEPT_VINTR))
2441			svm_clear_vintr(svm);
2442
2443		enable_gif(svm);
2444		if (svm->vcpu.arch.smi_pending ||
2445		    svm->vcpu.arch.nmi_pending ||
2446		    kvm_cpu_has_injectable_intr(&svm->vcpu) ||
2447		    kvm_apic_has_pending_init_or_sipi(&svm->vcpu))
2448			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2449	} else {
2450		disable_gif(svm);
2451
2452		/*
2453		 * After a CLGI no interrupts should come.  But if vGIF is
2454		 * in use, we still rely on the VINTR intercept (rather than
2455		 * STGI) to detect an open interrupt window.
2456		*/
2457		if (!vgif)
2458			svm_clear_vintr(svm);
2459	}
2460}
2461
2462static int stgi_interception(struct kvm_vcpu *vcpu)
2463{
2464	int ret;
2465
2466	if (nested_svm_check_permissions(vcpu))
2467		return 1;
2468
2469	ret = kvm_skip_emulated_instruction(vcpu);
2470	svm_set_gif(to_svm(vcpu), true);
2471	return ret;
2472}
2473
2474static int clgi_interception(struct kvm_vcpu *vcpu)
2475{
2476	int ret;
2477
2478	if (nested_svm_check_permissions(vcpu))
2479		return 1;
2480
2481	ret = kvm_skip_emulated_instruction(vcpu);
2482	svm_set_gif(to_svm(vcpu), false);
2483	return ret;
2484}
2485
2486static int invlpga_interception(struct kvm_vcpu *vcpu)
2487{
2488	gva_t gva = kvm_rax_read(vcpu);
2489	u32 asid = kvm_rcx_read(vcpu);
2490
2491	/* FIXME: Handle an address size prefix. */
2492	if (!is_long_mode(vcpu))
2493		gva = (u32)gva;
2494
2495	trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2496
2497	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2498	kvm_mmu_invlpg(vcpu, gva);
2499
2500	return kvm_skip_emulated_instruction(vcpu);
2501}
2502
2503static int skinit_interception(struct kvm_vcpu *vcpu)
2504{
2505	trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2506
2507	kvm_queue_exception(vcpu, UD_VECTOR);
2508	return 1;
2509}
2510
2511static int task_switch_interception(struct kvm_vcpu *vcpu)
2512{
2513	struct vcpu_svm *svm = to_svm(vcpu);
2514	u16 tss_selector;
2515	int reason;
2516	int int_type = svm->vmcb->control.exit_int_info &
2517		SVM_EXITINTINFO_TYPE_MASK;
2518	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2519	uint32_t type =
2520		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2521	uint32_t idt_v =
2522		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2523	bool has_error_code = false;
2524	u32 error_code = 0;
2525
2526	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2527
2528	if (svm->vmcb->control.exit_info_2 &
2529	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2530		reason = TASK_SWITCH_IRET;
2531	else if (svm->vmcb->control.exit_info_2 &
2532		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2533		reason = TASK_SWITCH_JMP;
2534	else if (idt_v)
2535		reason = TASK_SWITCH_GATE;
2536	else
2537		reason = TASK_SWITCH_CALL;
2538
2539	if (reason == TASK_SWITCH_GATE) {
2540		switch (type) {
2541		case SVM_EXITINTINFO_TYPE_NMI:
2542			vcpu->arch.nmi_injected = false;
2543			break;
2544		case SVM_EXITINTINFO_TYPE_EXEPT:
2545			if (svm->vmcb->control.exit_info_2 &
2546			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2547				has_error_code = true;
2548				error_code =
2549					(u32)svm->vmcb->control.exit_info_2;
2550			}
2551			kvm_clear_exception_queue(vcpu);
2552			break;
2553		case SVM_EXITINTINFO_TYPE_INTR:
2554		case SVM_EXITINTINFO_TYPE_SOFT:
2555			kvm_clear_interrupt_queue(vcpu);
2556			break;
2557		default:
2558			break;
2559		}
2560	}
2561
2562	if (reason != TASK_SWITCH_GATE ||
2563	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2564	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2565	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2566		if (!svm_skip_emulated_instruction(vcpu))
2567			return 0;
2568	}
2569
2570	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2571		int_vec = -1;
2572
2573	return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2574			       has_error_code, error_code);
2575}
2576
2577static void svm_clr_iret_intercept(struct vcpu_svm *svm)
2578{
2579	if (!sev_es_guest(svm->vcpu.kvm))
2580		svm_clr_intercept(svm, INTERCEPT_IRET);
2581}
2582
2583static void svm_set_iret_intercept(struct vcpu_svm *svm)
2584{
2585	if (!sev_es_guest(svm->vcpu.kvm))
2586		svm_set_intercept(svm, INTERCEPT_IRET);
2587}
2588
2589static int iret_interception(struct kvm_vcpu *vcpu)
2590{
2591	struct vcpu_svm *svm = to_svm(vcpu);
2592
2593	WARN_ON_ONCE(sev_es_guest(vcpu->kvm));
2594
2595	++vcpu->stat.nmi_window_exits;
2596	svm->awaiting_iret_completion = true;
2597
2598	svm_clr_iret_intercept(svm);
2599	svm->nmi_iret_rip = kvm_rip_read(vcpu);
2600
2601	kvm_make_request(KVM_REQ_EVENT, vcpu);
2602	return 1;
2603}
2604
2605static int invlpg_interception(struct kvm_vcpu *vcpu)
2606{
2607	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2608		return kvm_emulate_instruction(vcpu, 0);
2609
2610	kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2611	return kvm_skip_emulated_instruction(vcpu);
2612}
2613
2614static int emulate_on_interception(struct kvm_vcpu *vcpu)
2615{
2616	return kvm_emulate_instruction(vcpu, 0);
2617}
2618
2619static int rsm_interception(struct kvm_vcpu *vcpu)
2620{
2621	return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2622}
2623
2624static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2625					    unsigned long val)
2626{
2627	struct vcpu_svm *svm = to_svm(vcpu);
2628	unsigned long cr0 = vcpu->arch.cr0;
2629	bool ret = false;
2630
2631	if (!is_guest_mode(vcpu) ||
2632	    (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2633		return false;
2634
2635	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2636	val &= ~SVM_CR0_SELECTIVE_MASK;
2637
2638	if (cr0 ^ val) {
2639		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2640		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2641	}
2642
2643	return ret;
2644}
2645
2646#define CR_VALID (1ULL << 63)
2647
2648static int cr_interception(struct kvm_vcpu *vcpu)
2649{
2650	struct vcpu_svm *svm = to_svm(vcpu);
2651	int reg, cr;
2652	unsigned long val;
2653	int err;
2654
2655	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2656		return emulate_on_interception(vcpu);
2657
2658	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2659		return emulate_on_interception(vcpu);
2660
2661	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2662	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2663		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2664	else
2665		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2666
2667	err = 0;
2668	if (cr >= 16) { /* mov to cr */
2669		cr -= 16;
2670		val = kvm_register_read(vcpu, reg);
2671		trace_kvm_cr_write(cr, val);
2672		switch (cr) {
2673		case 0:
2674			if (!check_selective_cr0_intercepted(vcpu, val))
2675				err = kvm_set_cr0(vcpu, val);
2676			else
2677				return 1;
2678
2679			break;
2680		case 3:
2681			err = kvm_set_cr3(vcpu, val);
2682			break;
2683		case 4:
2684			err = kvm_set_cr4(vcpu, val);
2685			break;
2686		case 8:
2687			err = kvm_set_cr8(vcpu, val);
2688			break;
2689		default:
2690			WARN(1, "unhandled write to CR%d", cr);
2691			kvm_queue_exception(vcpu, UD_VECTOR);
2692			return 1;
2693		}
2694	} else { /* mov from cr */
2695		switch (cr) {
2696		case 0:
2697			val = kvm_read_cr0(vcpu);
2698			break;
2699		case 2:
2700			val = vcpu->arch.cr2;
2701			break;
2702		case 3:
2703			val = kvm_read_cr3(vcpu);
2704			break;
2705		case 4:
2706			val = kvm_read_cr4(vcpu);
2707			break;
2708		case 8:
2709			val = kvm_get_cr8(vcpu);
2710			break;
2711		default:
2712			WARN(1, "unhandled read from CR%d", cr);
2713			kvm_queue_exception(vcpu, UD_VECTOR);
2714			return 1;
2715		}
2716		kvm_register_write(vcpu, reg, val);
2717		trace_kvm_cr_read(cr, val);
2718	}
2719	return kvm_complete_insn_gp(vcpu, err);
2720}
2721
2722static int cr_trap(struct kvm_vcpu *vcpu)
2723{
2724	struct vcpu_svm *svm = to_svm(vcpu);
2725	unsigned long old_value, new_value;
2726	unsigned int cr;
2727	int ret = 0;
2728
2729	new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2730
2731	cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2732	switch (cr) {
2733	case 0:
2734		old_value = kvm_read_cr0(vcpu);
2735		svm_set_cr0(vcpu, new_value);
2736
2737		kvm_post_set_cr0(vcpu, old_value, new_value);
2738		break;
2739	case 4:
2740		old_value = kvm_read_cr4(vcpu);
2741		svm_set_cr4(vcpu, new_value);
2742
2743		kvm_post_set_cr4(vcpu, old_value, new_value);
2744		break;
2745	case 8:
2746		ret = kvm_set_cr8(vcpu, new_value);
2747		break;
2748	default:
2749		WARN(1, "unhandled CR%d write trap", cr);
2750		kvm_queue_exception(vcpu, UD_VECTOR);
2751		return 1;
2752	}
2753
2754	return kvm_complete_insn_gp(vcpu, ret);
2755}
2756
2757static int dr_interception(struct kvm_vcpu *vcpu)
2758{
2759	struct vcpu_svm *svm = to_svm(vcpu);
2760	int reg, dr;
 
2761	int err = 0;
2762
2763	/*
2764	 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT
2765	 * for DR7 write only. KVM cannot change DR7 (always swapped as type 'A') so return early.
2766	 */
2767	if (sev_es_guest(vcpu->kvm))
2768		return 1;
2769
2770	if (vcpu->guest_debug == 0) {
2771		/*
2772		 * No more DR vmexits; force a reload of the debug registers
2773		 * and reenter on this instruction.  The next vmexit will
2774		 * retrieve the full state of the debug registers.
2775		 */
2776		clr_dr_intercepts(svm);
2777		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2778		return 1;
2779	}
2780
2781	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2782		return emulate_on_interception(vcpu);
2783
2784	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2785	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2786	if (dr >= 16) { /* mov to DRn  */
2787		dr -= 16;
2788		err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg));
 
2789	} else {
2790		kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr));
 
2791	}
2792
2793	return kvm_complete_insn_gp(vcpu, err);
2794}
2795
2796static int cr8_write_interception(struct kvm_vcpu *vcpu)
2797{
2798	int r;
2799
2800	u8 cr8_prev = kvm_get_cr8(vcpu);
2801	/* instruction emulation calls kvm_set_cr8() */
2802	r = cr_interception(vcpu);
2803	if (lapic_in_kernel(vcpu))
2804		return r;
2805	if (cr8_prev <= kvm_get_cr8(vcpu))
2806		return r;
2807	vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2808	return 0;
2809}
2810
2811static int efer_trap(struct kvm_vcpu *vcpu)
2812{
2813	struct msr_data msr_info;
2814	int ret;
2815
2816	/*
2817	 * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2818	 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2819	 * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2820	 * the guest doesn't have X86_FEATURE_SVM.
2821	 */
2822	msr_info.host_initiated = false;
2823	msr_info.index = MSR_EFER;
2824	msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2825	ret = kvm_set_msr_common(vcpu, &msr_info);
2826
2827	return kvm_complete_insn_gp(vcpu, ret);
2828}
2829
2830static int svm_get_feature_msr(u32 msr, u64 *data)
2831{
2832	*data = 0;
2833
2834	switch (msr) {
2835	case MSR_AMD64_DE_CFG:
2836		if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
2837			*data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE;
2838		break;
2839	default:
2840		return KVM_MSR_RET_UNSUPPORTED;
2841	}
2842
2843	return 0;
2844}
2845
2846static bool
2847sev_es_prevent_msr_access(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2848{
2849	return sev_es_guest(vcpu->kvm) &&
2850	       vcpu->arch.guest_state_protected &&
2851	       svm_msrpm_offset(msr_info->index) != MSR_INVALID &&
2852	       !msr_write_intercepted(vcpu, msr_info->index);
2853}
2854
2855static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2856{
2857	struct vcpu_svm *svm = to_svm(vcpu);
2858
2859	if (sev_es_prevent_msr_access(vcpu, msr_info)) {
2860		msr_info->data = 0;
2861		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
2862	}
2863
2864	switch (msr_info->index) {
2865	case MSR_AMD64_TSC_RATIO:
2866		if (!msr_info->host_initiated &&
2867		    !guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR))
2868			return 1;
2869		msr_info->data = svm->tsc_ratio_msr;
2870		break;
2871	case MSR_STAR:
2872		msr_info->data = svm->vmcb01.ptr->save.star;
2873		break;
2874#ifdef CONFIG_X86_64
2875	case MSR_LSTAR:
2876		msr_info->data = svm->vmcb01.ptr->save.lstar;
2877		break;
2878	case MSR_CSTAR:
2879		msr_info->data = svm->vmcb01.ptr->save.cstar;
2880		break;
2881	case MSR_GS_BASE:
2882		msr_info->data = svm->vmcb01.ptr->save.gs.base;
2883		break;
2884	case MSR_FS_BASE:
2885		msr_info->data = svm->vmcb01.ptr->save.fs.base;
2886		break;
2887	case MSR_KERNEL_GS_BASE:
2888		msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2889		break;
2890	case MSR_SYSCALL_MASK:
2891		msr_info->data = svm->vmcb01.ptr->save.sfmask;
2892		break;
2893#endif
2894	case MSR_IA32_SYSENTER_CS:
2895		msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2896		break;
2897	case MSR_IA32_SYSENTER_EIP:
2898		msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2899		if (guest_cpuid_is_intel_compatible(vcpu))
2900			msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2901		break;
2902	case MSR_IA32_SYSENTER_ESP:
2903		msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2904		if (guest_cpuid_is_intel_compatible(vcpu))
2905			msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2906		break;
2907	case MSR_TSC_AUX:
2908		msr_info->data = svm->tsc_aux;
2909		break;
2910	case MSR_IA32_DEBUGCTLMSR:
2911		msr_info->data = svm_get_lbr_vmcb(svm)->save.dbgctl;
2912		break;
2913	case MSR_IA32_LASTBRANCHFROMIP:
2914		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_from;
2915		break;
2916	case MSR_IA32_LASTBRANCHTOIP:
2917		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_to;
2918		break;
2919	case MSR_IA32_LASTINTFROMIP:
2920		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_from;
2921		break;
2922	case MSR_IA32_LASTINTTOIP:
2923		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_to;
2924		break;
2925	case MSR_VM_HSAVE_PA:
2926		msr_info->data = svm->nested.hsave_msr;
2927		break;
2928	case MSR_VM_CR:
2929		msr_info->data = svm->nested.vm_cr_msr;
2930		break;
2931	case MSR_IA32_SPEC_CTRL:
2932		if (!msr_info->host_initiated &&
2933		    !guest_has_spec_ctrl_msr(vcpu))
2934			return 1;
2935
2936		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2937			msr_info->data = svm->vmcb->save.spec_ctrl;
2938		else
2939			msr_info->data = svm->spec_ctrl;
2940		break;
2941	case MSR_AMD64_VIRT_SPEC_CTRL:
2942		if (!msr_info->host_initiated &&
2943		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2944			return 1;
2945
2946		msr_info->data = svm->virt_spec_ctrl;
2947		break;
2948	case MSR_F15H_IC_CFG: {
2949
2950		int family, model;
2951
2952		family = guest_cpuid_family(vcpu);
2953		model  = guest_cpuid_model(vcpu);
2954
2955		if (family < 0 || model < 0)
2956			return kvm_get_msr_common(vcpu, msr_info);
2957
2958		msr_info->data = 0;
2959
2960		if (family == 0x15 &&
2961		    (model >= 0x2 && model < 0x20))
2962			msr_info->data = 0x1E;
2963		}
2964		break;
2965	case MSR_AMD64_DE_CFG:
2966		msr_info->data = svm->msr_decfg;
2967		break;
2968	default:
2969		return kvm_get_msr_common(vcpu, msr_info);
2970	}
2971	return 0;
2972}
2973
2974static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
2975{
2976	struct vcpu_svm *svm = to_svm(vcpu);
2977	if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb))
2978		return kvm_complete_insn_gp(vcpu, err);
2979
2980	ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1);
2981	ghcb_set_sw_exit_info_2(svm->sev_es.ghcb,
2982				X86_TRAP_GP |
2983				SVM_EVTINJ_TYPE_EXEPT |
2984				SVM_EVTINJ_VALID);
2985	return 1;
2986}
2987
2988static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2989{
2990	struct vcpu_svm *svm = to_svm(vcpu);
2991	int svm_dis, chg_mask;
2992
2993	if (data & ~SVM_VM_CR_VALID_MASK)
2994		return 1;
2995
2996	chg_mask = SVM_VM_CR_VALID_MASK;
2997
2998	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2999		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
3000
3001	svm->nested.vm_cr_msr &= ~chg_mask;
3002	svm->nested.vm_cr_msr |= (data & chg_mask);
3003
3004	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
3005
3006	/* check for svm_disable while efer.svme is set */
3007	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
3008		return 1;
3009
3010	return 0;
3011}
3012
3013static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3014{
3015	struct vcpu_svm *svm = to_svm(vcpu);
3016	int ret = 0;
3017
3018	u32 ecx = msr->index;
3019	u64 data = msr->data;
3020
3021	if (sev_es_prevent_msr_access(vcpu, msr))
3022		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
3023
3024	switch (ecx) {
3025	case MSR_AMD64_TSC_RATIO:
3026
3027		if (!guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR)) {
3028
3029			if (!msr->host_initiated)
3030				return 1;
3031			/*
3032			 * In case TSC scaling is not enabled, always
3033			 * leave this MSR at the default value.
3034			 *
3035			 * Due to bug in qemu 6.2.0, it would try to set
3036			 * this msr to 0 if tsc scaling is not enabled.
3037			 * Ignore this value as well.
3038			 */
3039			if (data != 0 && data != svm->tsc_ratio_msr)
3040				return 1;
3041			break;
3042		}
3043
3044		if (data & SVM_TSC_RATIO_RSVD)
3045			return 1;
3046
3047		svm->tsc_ratio_msr = data;
3048
3049		if (guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR) &&
3050		    is_guest_mode(vcpu))
3051			nested_svm_update_tsc_ratio_msr(vcpu);
3052
3053		break;
3054	case MSR_IA32_CR_PAT:
3055		ret = kvm_set_msr_common(vcpu, msr);
3056		if (ret)
3057			break;
3058
3059		svm->vmcb01.ptr->save.g_pat = data;
3060		if (is_guest_mode(vcpu))
3061			nested_vmcb02_compute_g_pat(svm);
3062		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3063		break;
3064	case MSR_IA32_SPEC_CTRL:
3065		if (!msr->host_initiated &&
3066		    !guest_has_spec_ctrl_msr(vcpu))
3067			return 1;
3068
3069		if (kvm_spec_ctrl_test_value(data))
3070			return 1;
3071
3072		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3073			svm->vmcb->save.spec_ctrl = data;
3074		else
3075			svm->spec_ctrl = data;
3076		if (!data)
3077			break;
3078
3079		/*
3080		 * For non-nested:
3081		 * When it's written (to non-zero) for the first time, pass
3082		 * it through.
3083		 *
3084		 * For nested:
3085		 * The handling of the MSR bitmap for L2 guests is done in
3086		 * nested_svm_vmrun_msrpm.
3087		 * We update the L1 MSR bit as well since it will end up
3088		 * touching the MSR anyway now.
3089		 */
3090		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
3091		break;
3092	case MSR_AMD64_VIRT_SPEC_CTRL:
3093		if (!msr->host_initiated &&
3094		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
3095			return 1;
3096
3097		if (data & ~SPEC_CTRL_SSBD)
3098			return 1;
3099
3100		svm->virt_spec_ctrl = data;
3101		break;
3102	case MSR_STAR:
3103		svm->vmcb01.ptr->save.star = data;
3104		break;
3105#ifdef CONFIG_X86_64
3106	case MSR_LSTAR:
3107		svm->vmcb01.ptr->save.lstar = data;
3108		break;
3109	case MSR_CSTAR:
3110		svm->vmcb01.ptr->save.cstar = data;
3111		break;
3112	case MSR_GS_BASE:
3113		svm->vmcb01.ptr->save.gs.base = data;
3114		break;
3115	case MSR_FS_BASE:
3116		svm->vmcb01.ptr->save.fs.base = data;
3117		break;
3118	case MSR_KERNEL_GS_BASE:
3119		svm->vmcb01.ptr->save.kernel_gs_base = data;
3120		break;
3121	case MSR_SYSCALL_MASK:
3122		svm->vmcb01.ptr->save.sfmask = data;
3123		break;
3124#endif
3125	case MSR_IA32_SYSENTER_CS:
3126		svm->vmcb01.ptr->save.sysenter_cs = data;
3127		break;
3128	case MSR_IA32_SYSENTER_EIP:
3129		svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
3130		/*
3131		 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
3132		 * when we spoof an Intel vendor ID (for cross vendor migration).
3133		 * In this case we use this intercept to track the high
3134		 * 32 bit part of these msrs to support Intel's
3135		 * implementation of SYSENTER/SYSEXIT.
3136		 */
3137		svm->sysenter_eip_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0;
3138		break;
3139	case MSR_IA32_SYSENTER_ESP:
3140		svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
3141		svm->sysenter_esp_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0;
3142		break;
3143	case MSR_TSC_AUX:
3144		/*
3145		 * TSC_AUX is always virtualized for SEV-ES guests when the
3146		 * feature is available. The user return MSR support is not
3147		 * required in this case because TSC_AUX is restored on #VMEXIT
3148		 * from the host save area (which has been initialized in
3149		 * svm_enable_virtualization_cpu()).
3150		 */
3151		if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm))
3152			break;
3153
3154		/*
3155		 * TSC_AUX is usually changed only during boot and never read
3156		 * directly.  Intercept TSC_AUX instead of exposing it to the
3157		 * guest via direct_access_msrs, and switch it via user return.
3158		 */
3159		preempt_disable();
3160		ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
3161		preempt_enable();
3162		if (ret)
3163			break;
3164
3165		svm->tsc_aux = data;
3166		break;
3167	case MSR_IA32_DEBUGCTLMSR:
3168		if (!lbrv) {
3169			kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3170			break;
3171		}
3172
3173		/*
3174		 * AMD changed the architectural behavior of bits 5:2.  On CPUs
3175		 * without BusLockTrap, bits 5:2 control "external pins", but
3176		 * on CPUs that support BusLockDetect, bit 2 enables BusLockTrap
3177		 * and bits 5:3 are reserved-to-zero.  Sadly, old KVM allowed
3178		 * the guest to set bits 5:2 despite not actually virtualizing
3179		 * Performance-Monitoring/Breakpoint external pins.  Drop bits
3180		 * 5:2 for backwards compatibility.
3181		 */
3182		data &= ~GENMASK(5, 2);
3183
3184		/*
3185		 * Suppress BTF as KVM doesn't virtualize BTF, but there's no
3186		 * way to communicate lack of support to the guest.
3187		 */
3188		if (data & DEBUGCTLMSR_BTF) {
3189			kvm_pr_unimpl_wrmsr(vcpu, MSR_IA32_DEBUGCTLMSR, data);
3190			data &= ~DEBUGCTLMSR_BTF;
3191		}
3192
3193		if (data & DEBUGCTL_RESERVED_BITS)
3194			return 1;
3195
3196		svm_get_lbr_vmcb(svm)->save.dbgctl = data;
3197		svm_update_lbrv(vcpu);
3198		break;
3199	case MSR_VM_HSAVE_PA:
3200		/*
3201		 * Old kernels did not validate the value written to
3202		 * MSR_VM_HSAVE_PA.  Allow KVM_SET_MSR to set an invalid
3203		 * value to allow live migrating buggy or malicious guests
3204		 * originating from those kernels.
3205		 */
3206		if (!msr->host_initiated && !page_address_valid(vcpu, data))
3207			return 1;
3208
3209		svm->nested.hsave_msr = data & PAGE_MASK;
3210		break;
3211	case MSR_VM_CR:
3212		return svm_set_vm_cr(vcpu, data);
3213	case MSR_VM_IGNNE:
3214		kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3215		break;
3216	case MSR_AMD64_DE_CFG: {
3217		u64 supported_de_cfg;
3218
3219		if (svm_get_feature_msr(ecx, &supported_de_cfg))
 
3220			return 1;
3221
3222		if (data & ~supported_de_cfg)
 
 
 
 
 
3223			return 1;
3224
3225		svm->msr_decfg = data;
3226		break;
3227	}
3228	default:
3229		return kvm_set_msr_common(vcpu, msr);
3230	}
3231	return ret;
3232}
3233
3234static int msr_interception(struct kvm_vcpu *vcpu)
3235{
3236	if (to_svm(vcpu)->vmcb->control.exit_info_1)
3237		return kvm_emulate_wrmsr(vcpu);
3238	else
3239		return kvm_emulate_rdmsr(vcpu);
3240}
3241
3242static int interrupt_window_interception(struct kvm_vcpu *vcpu)
3243{
3244	kvm_make_request(KVM_REQ_EVENT, vcpu);
3245	svm_clear_vintr(to_svm(vcpu));
3246
3247	/*
3248	 * If not running nested, for AVIC, the only reason to end up here is ExtINTs.
3249	 * In this case AVIC was temporarily disabled for
3250	 * requesting the IRQ window and we have to re-enable it.
3251	 *
3252	 * If running nested, still remove the VM wide AVIC inhibit to
3253	 * support case in which the interrupt window was requested when the
3254	 * vCPU was not running nested.
3255
3256	 * All vCPUs which run still run nested, will remain to have their
3257	 * AVIC still inhibited due to per-cpu AVIC inhibition.
3258	 */
3259	kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3260
3261	++vcpu->stat.irq_window_exits;
3262	return 1;
3263}
3264
3265static int pause_interception(struct kvm_vcpu *vcpu)
3266{
3267	bool in_kernel;
3268	/*
3269	 * CPL is not made available for an SEV-ES guest, therefore
3270	 * vcpu->arch.preempted_in_kernel can never be true.  Just
3271	 * set in_kernel to false as well.
3272	 */
3273	in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3274
3275	grow_ple_window(vcpu);
3276
3277	kvm_vcpu_on_spin(vcpu, in_kernel);
3278	return kvm_skip_emulated_instruction(vcpu);
3279}
3280
3281static int invpcid_interception(struct kvm_vcpu *vcpu)
3282{
3283	struct vcpu_svm *svm = to_svm(vcpu);
3284	unsigned long type;
3285	gva_t gva;
3286
3287	if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
3288		kvm_queue_exception(vcpu, UD_VECTOR);
3289		return 1;
3290	}
3291
3292	/*
3293	 * For an INVPCID intercept:
3294	 * EXITINFO1 provides the linear address of the memory operand.
3295	 * EXITINFO2 provides the contents of the register operand.
3296	 */
3297	type = svm->vmcb->control.exit_info_2;
3298	gva = svm->vmcb->control.exit_info_1;
3299
3300	return kvm_handle_invpcid(vcpu, type, gva);
3301}
3302
3303static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3304	[SVM_EXIT_READ_CR0]			= cr_interception,
3305	[SVM_EXIT_READ_CR3]			= cr_interception,
3306	[SVM_EXIT_READ_CR4]			= cr_interception,
3307	[SVM_EXIT_READ_CR8]			= cr_interception,
3308	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
3309	[SVM_EXIT_WRITE_CR0]			= cr_interception,
3310	[SVM_EXIT_WRITE_CR3]			= cr_interception,
3311	[SVM_EXIT_WRITE_CR4]			= cr_interception,
3312	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
3313	[SVM_EXIT_READ_DR0]			= dr_interception,
3314	[SVM_EXIT_READ_DR1]			= dr_interception,
3315	[SVM_EXIT_READ_DR2]			= dr_interception,
3316	[SVM_EXIT_READ_DR3]			= dr_interception,
3317	[SVM_EXIT_READ_DR4]			= dr_interception,
3318	[SVM_EXIT_READ_DR5]			= dr_interception,
3319	[SVM_EXIT_READ_DR6]			= dr_interception,
3320	[SVM_EXIT_READ_DR7]			= dr_interception,
3321	[SVM_EXIT_WRITE_DR0]			= dr_interception,
3322	[SVM_EXIT_WRITE_DR1]			= dr_interception,
3323	[SVM_EXIT_WRITE_DR2]			= dr_interception,
3324	[SVM_EXIT_WRITE_DR3]			= dr_interception,
3325	[SVM_EXIT_WRITE_DR4]			= dr_interception,
3326	[SVM_EXIT_WRITE_DR5]			= dr_interception,
3327	[SVM_EXIT_WRITE_DR6]			= dr_interception,
3328	[SVM_EXIT_WRITE_DR7]			= dr_interception,
3329	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
3330	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
3331	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
3332	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
3333	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
3334	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
3335	[SVM_EXIT_EXCP_BASE + GP_VECTOR]	= gp_interception,
3336	[SVM_EXIT_INTR]				= intr_interception,
3337	[SVM_EXIT_NMI]				= nmi_interception,
3338	[SVM_EXIT_SMI]				= smi_interception,
3339	[SVM_EXIT_VINTR]			= interrupt_window_interception,
3340	[SVM_EXIT_RDPMC]			= kvm_emulate_rdpmc,
3341	[SVM_EXIT_CPUID]			= kvm_emulate_cpuid,
3342	[SVM_EXIT_IRET]                         = iret_interception,
3343	[SVM_EXIT_INVD]                         = kvm_emulate_invd,
3344	[SVM_EXIT_PAUSE]			= pause_interception,
3345	[SVM_EXIT_HLT]				= kvm_emulate_halt,
3346	[SVM_EXIT_INVLPG]			= invlpg_interception,
3347	[SVM_EXIT_INVLPGA]			= invlpga_interception,
3348	[SVM_EXIT_IOIO]				= io_interception,
3349	[SVM_EXIT_MSR]				= msr_interception,
3350	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
3351	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
3352	[SVM_EXIT_VMRUN]			= vmrun_interception,
3353	[SVM_EXIT_VMMCALL]			= kvm_emulate_hypercall,
3354	[SVM_EXIT_VMLOAD]			= vmload_interception,
3355	[SVM_EXIT_VMSAVE]			= vmsave_interception,
3356	[SVM_EXIT_STGI]				= stgi_interception,
3357	[SVM_EXIT_CLGI]				= clgi_interception,
3358	[SVM_EXIT_SKINIT]			= skinit_interception,
3359	[SVM_EXIT_RDTSCP]			= kvm_handle_invalid_op,
3360	[SVM_EXIT_WBINVD]                       = kvm_emulate_wbinvd,
3361	[SVM_EXIT_MONITOR]			= kvm_emulate_monitor,
3362	[SVM_EXIT_MWAIT]			= kvm_emulate_mwait,
3363	[SVM_EXIT_XSETBV]			= kvm_emulate_xsetbv,
3364	[SVM_EXIT_RDPRU]			= kvm_handle_invalid_op,
3365	[SVM_EXIT_EFER_WRITE_TRAP]		= efer_trap,
3366	[SVM_EXIT_CR0_WRITE_TRAP]		= cr_trap,
3367	[SVM_EXIT_CR4_WRITE_TRAP]		= cr_trap,
3368	[SVM_EXIT_CR8_WRITE_TRAP]		= cr_trap,
3369	[SVM_EXIT_INVPCID]                      = invpcid_interception,
3370	[SVM_EXIT_NPF]				= npf_interception,
3371	[SVM_EXIT_RSM]                          = rsm_interception,
3372	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
3373	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
3374#ifdef CONFIG_KVM_AMD_SEV
3375	[SVM_EXIT_VMGEXIT]			= sev_handle_vmgexit,
3376#endif
3377};
3378
3379static void dump_vmcb(struct kvm_vcpu *vcpu)
3380{
3381	struct vcpu_svm *svm = to_svm(vcpu);
3382	struct vmcb_control_area *control = &svm->vmcb->control;
3383	struct vmcb_save_area *save = &svm->vmcb->save;
3384	struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3385
3386	if (!dump_invalid_vmcb) {
3387		pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3388		return;
3389	}
3390
3391	pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3392	       svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3393	pr_err("VMCB Control Area:\n");
3394	pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3395	pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3396	pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3397	pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3398	pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3399	pr_err("%-20s%08x %08x\n", "intercepts:",
3400              control->intercepts[INTERCEPT_WORD3],
3401	       control->intercepts[INTERCEPT_WORD4]);
3402	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3403	pr_err("%-20s%d\n", "pause filter threshold:",
3404	       control->pause_filter_thresh);
3405	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3406	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3407	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3408	pr_err("%-20s%d\n", "asid:", control->asid);
3409	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3410	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3411	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3412	pr_err("%-20s%08x\n", "int_state:", control->int_state);
3413	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3414	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3415	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3416	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3417	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3418	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3419	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3420	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3421	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3422	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3423	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3424	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3425	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3426	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3427	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3428	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3429	pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3430	pr_err("VMCB State Save Area:\n");
3431	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3432	       "es:",
3433	       save->es.selector, save->es.attrib,
3434	       save->es.limit, save->es.base);
3435	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3436	       "cs:",
3437	       save->cs.selector, save->cs.attrib,
3438	       save->cs.limit, save->cs.base);
3439	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3440	       "ss:",
3441	       save->ss.selector, save->ss.attrib,
3442	       save->ss.limit, save->ss.base);
3443	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3444	       "ds:",
3445	       save->ds.selector, save->ds.attrib,
3446	       save->ds.limit, save->ds.base);
3447	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3448	       "fs:",
3449	       save01->fs.selector, save01->fs.attrib,
3450	       save01->fs.limit, save01->fs.base);
3451	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3452	       "gs:",
3453	       save01->gs.selector, save01->gs.attrib,
3454	       save01->gs.limit, save01->gs.base);
3455	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3456	       "gdtr:",
3457	       save->gdtr.selector, save->gdtr.attrib,
3458	       save->gdtr.limit, save->gdtr.base);
3459	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3460	       "ldtr:",
3461	       save01->ldtr.selector, save01->ldtr.attrib,
3462	       save01->ldtr.limit, save01->ldtr.base);
3463	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3464	       "idtr:",
3465	       save->idtr.selector, save->idtr.attrib,
3466	       save->idtr.limit, save->idtr.base);
3467	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3468	       "tr:",
3469	       save01->tr.selector, save01->tr.attrib,
3470	       save01->tr.limit, save01->tr.base);
3471	pr_err("vmpl: %d   cpl:  %d               efer:          %016llx\n",
3472	       save->vmpl, save->cpl, save->efer);
3473	pr_err("%-15s %016llx %-13s %016llx\n",
3474	       "cr0:", save->cr0, "cr2:", save->cr2);
3475	pr_err("%-15s %016llx %-13s %016llx\n",
3476	       "cr3:", save->cr3, "cr4:", save->cr4);
3477	pr_err("%-15s %016llx %-13s %016llx\n",
3478	       "dr6:", save->dr6, "dr7:", save->dr7);
3479	pr_err("%-15s %016llx %-13s %016llx\n",
3480	       "rip:", save->rip, "rflags:", save->rflags);
3481	pr_err("%-15s %016llx %-13s %016llx\n",
3482	       "rsp:", save->rsp, "rax:", save->rax);
3483	pr_err("%-15s %016llx %-13s %016llx\n",
3484	       "star:", save01->star, "lstar:", save01->lstar);
3485	pr_err("%-15s %016llx %-13s %016llx\n",
3486	       "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3487	pr_err("%-15s %016llx %-13s %016llx\n",
3488	       "kernel_gs_base:", save01->kernel_gs_base,
3489	       "sysenter_cs:", save01->sysenter_cs);
3490	pr_err("%-15s %016llx %-13s %016llx\n",
3491	       "sysenter_esp:", save01->sysenter_esp,
3492	       "sysenter_eip:", save01->sysenter_eip);
3493	pr_err("%-15s %016llx %-13s %016llx\n",
3494	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3495	pr_err("%-15s %016llx %-13s %016llx\n",
3496	       "br_from:", save->br_from, "br_to:", save->br_to);
3497	pr_err("%-15s %016llx %-13s %016llx\n",
3498	       "excp_from:", save->last_excp_from,
3499	       "excp_to:", save->last_excp_to);
3500}
3501
3502static bool svm_check_exit_valid(u64 exit_code)
3503{
3504	return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3505		svm_exit_handlers[exit_code]);
3506}
3507
3508static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3509{
3510	vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
3511	dump_vmcb(vcpu);
3512	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3513	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3514	vcpu->run->internal.ndata = 2;
3515	vcpu->run->internal.data[0] = exit_code;
3516	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3517	return 0;
3518}
3519
3520int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3521{
3522	if (!svm_check_exit_valid(exit_code))
3523		return svm_handle_invalid_exit(vcpu, exit_code);
3524
3525#ifdef CONFIG_MITIGATION_RETPOLINE
3526	if (exit_code == SVM_EXIT_MSR)
3527		return msr_interception(vcpu);
3528	else if (exit_code == SVM_EXIT_VINTR)
3529		return interrupt_window_interception(vcpu);
3530	else if (exit_code == SVM_EXIT_INTR)
3531		return intr_interception(vcpu);
3532	else if (exit_code == SVM_EXIT_HLT)
3533		return kvm_emulate_halt(vcpu);
3534	else if (exit_code == SVM_EXIT_NPF)
3535		return npf_interception(vcpu);
3536#endif
3537	return svm_exit_handlers[exit_code](vcpu);
3538}
3539
3540static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
3541			      u64 *info1, u64 *info2,
3542			      u32 *intr_info, u32 *error_code)
3543{
3544	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3545
3546	*reason = control->exit_code;
3547	*info1 = control->exit_info_1;
3548	*info2 = control->exit_info_2;
3549	*intr_info = control->exit_int_info;
3550	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3551	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3552		*error_code = control->exit_int_info_err;
3553	else
3554		*error_code = 0;
3555}
3556
3557static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3558{
3559	struct vcpu_svm *svm = to_svm(vcpu);
3560	struct kvm_run *kvm_run = vcpu->run;
3561	u32 exit_code = svm->vmcb->control.exit_code;
3562
3563	/* SEV-ES guests must use the CR write traps to track CR registers. */
3564	if (!sev_es_guest(vcpu->kvm)) {
3565		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3566			vcpu->arch.cr0 = svm->vmcb->save.cr0;
3567		if (npt_enabled)
3568			vcpu->arch.cr3 = svm->vmcb->save.cr3;
3569	}
3570
3571	if (is_guest_mode(vcpu)) {
3572		int vmexit;
3573
3574		trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
3575
3576		vmexit = nested_svm_exit_special(svm);
3577
3578		if (vmexit == NESTED_EXIT_CONTINUE)
3579			vmexit = nested_svm_exit_handled(svm);
3580
3581		if (vmexit == NESTED_EXIT_DONE)
3582			return 1;
3583	}
3584
3585	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3586		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3587		kvm_run->fail_entry.hardware_entry_failure_reason
3588			= svm->vmcb->control.exit_code;
3589		kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3590		dump_vmcb(vcpu);
3591		return 0;
3592	}
3593
3594	if (exit_fastpath != EXIT_FASTPATH_NONE)
3595		return 1;
3596
3597	return svm_invoke_exit_handler(vcpu, exit_code);
3598}
3599
3600static void pre_svm_run(struct kvm_vcpu *vcpu)
3601{
3602	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
3603	struct vcpu_svm *svm = to_svm(vcpu);
3604
3605	/*
3606	 * If the previous vmrun of the vmcb occurred on a different physical
3607	 * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
3608	 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3609	 */
3610	if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3611		svm->current_vmcb->asid_generation = 0;
3612		vmcb_mark_all_dirty(svm->vmcb);
3613		svm->current_vmcb->cpu = vcpu->cpu;
3614        }
3615
3616	if (sev_guest(vcpu->kvm))
3617		return pre_sev_run(svm, vcpu->cpu);
3618
3619	/* FIXME: handle wraparound of asid_generation */
3620	if (svm->current_vmcb->asid_generation != sd->asid_generation)
3621		new_asid(svm, sd);
3622}
3623
3624static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3625{
3626	struct vcpu_svm *svm = to_svm(vcpu);
3627
3628	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3629
3630	if (svm->nmi_l1_to_l2)
3631		return;
3632
3633	/*
3634	 * No need to manually track NMI masking when vNMI is enabled, hardware
3635	 * automatically sets V_NMI_BLOCKING_MASK as appropriate, including the
3636	 * case where software directly injects an NMI.
3637	 */
3638	if (!is_vnmi_enabled(svm)) {
3639		svm->nmi_masked = true;
3640		svm_set_iret_intercept(svm);
3641	}
3642	++vcpu->stat.nmi_injections;
3643}
3644
3645static bool svm_is_vnmi_pending(struct kvm_vcpu *vcpu)
3646{
3647	struct vcpu_svm *svm = to_svm(vcpu);
3648
3649	if (!is_vnmi_enabled(svm))
3650		return false;
3651
3652	return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK);
3653}
3654
3655static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu)
3656{
3657	struct vcpu_svm *svm = to_svm(vcpu);
3658
3659	if (!is_vnmi_enabled(svm))
3660		return false;
3661
3662	if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK)
3663		return false;
3664
3665	svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK;
3666	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
3667
3668	/*
3669	 * Because the pending NMI is serviced by hardware, KVM can't know when
3670	 * the NMI is "injected", but for all intents and purposes, passing the
3671	 * NMI off to hardware counts as injection.
3672	 */
3673	++vcpu->stat.nmi_injections;
3674
3675	return true;
3676}
3677
3678static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
3679{
3680	struct vcpu_svm *svm = to_svm(vcpu);
3681	u32 type;
3682
3683	if (vcpu->arch.interrupt.soft) {
3684		if (svm_update_soft_interrupt_rip(vcpu))
3685			return;
3686
3687		type = SVM_EVTINJ_TYPE_SOFT;
3688	} else {
3689		type = SVM_EVTINJ_TYPE_INTR;
3690	}
3691
3692	trace_kvm_inj_virq(vcpu->arch.interrupt.nr,
3693			   vcpu->arch.interrupt.soft, reinjected);
3694	++vcpu->stat.irq_injections;
3695
3696	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3697				       SVM_EVTINJ_VALID | type;
3698}
3699
3700void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
3701				     int trig_mode, int vector)
3702{
3703	/*
3704	 * apic->apicv_active must be read after vcpu->mode.
3705	 * Pairs with smp_store_release in vcpu_enter_guest.
3706	 */
3707	bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE);
3708
3709	/* Note, this is called iff the local APIC is in-kernel. */
3710	if (!READ_ONCE(vcpu->arch.apic->apicv_active)) {
3711		/* Process the interrupt via kvm_check_and_inject_events(). */
3712		kvm_make_request(KVM_REQ_EVENT, vcpu);
3713		kvm_vcpu_kick(vcpu);
3714		return;
3715	}
3716
3717	trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
3718	if (in_guest_mode) {
3719		/*
3720		 * Signal the doorbell to tell hardware to inject the IRQ.  If
3721		 * the vCPU exits the guest before the doorbell chimes, hardware
3722		 * will automatically process AVIC interrupts at the next VMRUN.
3723		 */
3724		avic_ring_doorbell(vcpu);
3725	} else {
3726		/*
3727		 * Wake the vCPU if it was blocking.  KVM will then detect the
3728		 * pending IRQ when checking if the vCPU has a wake event.
3729		 */
3730		kvm_vcpu_wake_up(vcpu);
3731	}
3732}
3733
3734static void svm_deliver_interrupt(struct kvm_lapic *apic,  int delivery_mode,
3735				  int trig_mode, int vector)
3736{
3737	kvm_lapic_set_irr(vector, apic);
3738
3739	/*
3740	 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in
3741	 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before
3742	 * the read of guest_mode.  This guarantees that either VMRUN will see
3743	 * and process the new vIRR entry, or that svm_complete_interrupt_delivery
3744	 * will signal the doorbell if the CPU has already entered the guest.
3745	 */
3746	smp_mb__after_atomic();
3747	svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector);
3748}
3749
3750static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3751{
3752	struct vcpu_svm *svm = to_svm(vcpu);
3753
3754	/*
3755	 * SEV-ES guests must always keep the CR intercepts cleared. CR
3756	 * tracking is done using the CR write traps.
3757	 */
3758	if (sev_es_guest(vcpu->kvm))
3759		return;
3760
3761	if (nested_svm_virtualize_tpr(vcpu))
3762		return;
3763
3764	svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3765
3766	if (irr == -1)
3767		return;
3768
3769	if (tpr >= irr)
3770		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3771}
3772
3773static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3774{
3775	struct vcpu_svm *svm = to_svm(vcpu);
3776
3777	if (is_vnmi_enabled(svm))
3778		return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK;
3779	else
3780		return svm->nmi_masked;
3781}
3782
3783static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3784{
3785	struct vcpu_svm *svm = to_svm(vcpu);
3786
3787	if (is_vnmi_enabled(svm)) {
3788		if (masked)
3789			svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK;
3790		else
3791			svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
3792
3793	} else {
3794		svm->nmi_masked = masked;
3795		if (masked)
3796			svm_set_iret_intercept(svm);
3797		else
3798			svm_clr_iret_intercept(svm);
3799	}
3800}
3801
3802bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3803{
3804	struct vcpu_svm *svm = to_svm(vcpu);
3805	struct vmcb *vmcb = svm->vmcb;
3806
3807	if (!gif_set(svm))
3808		return true;
3809
3810	if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3811		return false;
3812
3813	if (svm_get_nmi_mask(vcpu))
3814		return true;
3815
3816	return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK;
3817}
3818
3819static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3820{
3821	struct vcpu_svm *svm = to_svm(vcpu);
3822	if (svm->nested.nested_run_pending)
3823		return -EBUSY;
3824
3825	if (svm_nmi_blocked(vcpu))
3826		return 0;
3827
3828	/* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3829	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3830		return -EBUSY;
3831	return 1;
3832}
3833
3834bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3835{
3836	struct vcpu_svm *svm = to_svm(vcpu);
3837	struct vmcb *vmcb = svm->vmcb;
3838
3839	if (!gif_set(svm))
3840		return true;
3841
3842	if (is_guest_mode(vcpu)) {
3843		/* As long as interrupts are being delivered...  */
3844		if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3845		    ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3846		    : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3847			return true;
3848
3849		/* ... vmexits aren't blocked by the interrupt shadow  */
3850		if (nested_exit_on_intr(svm))
3851			return false;
3852	} else {
3853		if (!svm_get_if_flag(vcpu))
3854			return true;
3855	}
3856
3857	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3858}
3859
3860static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3861{
3862	struct vcpu_svm *svm = to_svm(vcpu);
3863
3864	if (svm->nested.nested_run_pending)
3865		return -EBUSY;
3866
3867	if (svm_interrupt_blocked(vcpu))
3868		return 0;
3869
3870	/*
3871	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3872	 * e.g. if the IRQ arrived asynchronously after checking nested events.
3873	 */
3874	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3875		return -EBUSY;
3876
3877	return 1;
3878}
3879
3880static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3881{
3882	struct vcpu_svm *svm = to_svm(vcpu);
3883
3884	/*
3885	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3886	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3887	 * get that intercept, this function will be called again though and
3888	 * we'll get the vintr intercept. However, if the vGIF feature is
3889	 * enabled, the STGI interception will not occur. Enable the irq
3890	 * window under the assumption that the hardware will set the GIF.
3891	 */
3892	if (vgif || gif_set(svm)) {
3893		/*
3894		 * IRQ window is not needed when AVIC is enabled,
3895		 * unless we have pending ExtINT since it cannot be injected
3896		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
3897		 * and fallback to injecting IRQ via V_IRQ.
3898		 *
3899		 * If running nested, AVIC is already locally inhibited
3900		 * on this vCPU, therefore there is no need to request
3901		 * the VM wide AVIC inhibition.
3902		 */
3903		if (!is_guest_mode(vcpu))
3904			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3905
3906		svm_set_vintr(svm);
3907	}
3908}
3909
3910static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3911{
3912	struct vcpu_svm *svm = to_svm(vcpu);
3913
3914	/*
3915	 * If NMIs are outright masked, i.e. the vCPU is already handling an
3916	 * NMI, and KVM has not yet intercepted an IRET, then there is nothing
3917	 * more to do at this time as KVM has already enabled IRET intercepts.
3918	 * If KVM has already intercepted IRET, then single-step over the IRET,
3919	 * as NMIs aren't architecturally unmasked until the IRET completes.
3920	 *
3921	 * If vNMI is enabled, KVM should never request an NMI window if NMIs
3922	 * are masked, as KVM allows at most one to-be-injected NMI and one
3923	 * pending NMI.  If two NMIs arrive simultaneously, KVM will inject one
3924	 * NMI and set V_NMI_PENDING for the other, but if and only if NMIs are
3925	 * unmasked.  KVM _will_ request an NMI window in some situations, e.g.
3926	 * if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately
3927	 * inject the NMI.  In those situations, KVM needs to single-step over
3928	 * the STI shadow or intercept STGI.
3929	 */
3930	if (svm_get_nmi_mask(vcpu)) {
3931		WARN_ON_ONCE(is_vnmi_enabled(svm));
3932
3933		if (!svm->awaiting_iret_completion)
3934			return; /* IRET will cause a vm exit */
3935	}
3936
3937	/*
3938	 * SEV-ES guests are responsible for signaling when a vCPU is ready to
3939	 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e.
3940	 * KVM can't intercept and single-step IRET to detect when NMIs are
3941	 * unblocked (architecturally speaking).  See SVM_VMGEXIT_NMI_COMPLETE.
3942	 *
3943	 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware
3944	 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not
3945	 * supported NAEs in the GHCB protocol.
3946	 */
3947	if (sev_es_guest(vcpu->kvm))
3948		return;
3949
3950	if (!gif_set(svm)) {
3951		if (vgif)
3952			svm_set_intercept(svm, INTERCEPT_STGI);
3953		return; /* STGI will cause a vm exit */
3954	}
3955
3956	/*
3957	 * Something prevents NMI from been injected. Single step over possible
3958	 * problem (IRET or exception injection or interrupt shadow)
3959	 */
3960	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3961	svm->nmi_singlestep = true;
3962	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3963}
3964
3965static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)
3966{
3967	struct vcpu_svm *svm = to_svm(vcpu);
3968
3969	/*
3970	 * Unlike VMX, SVM doesn't provide a way to flush only NPT TLB entries.
3971	 * A TLB flush for the current ASID flushes both "host" and "guest" TLB
3972	 * entries, and thus is a superset of Hyper-V's fine grained flushing.
3973	 */
3974	kvm_hv_vcpu_purge_flush_tlb(vcpu);
3975
3976	/*
3977	 * Flush only the current ASID even if the TLB flush was invoked via
3978	 * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
3979	 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3980	 * unconditionally does a TLB flush on both nested VM-Enter and nested
3981	 * VM-Exit (via kvm_mmu_reset_context()).
3982	 */
3983	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3984		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3985	else
3986		svm->current_vmcb->asid_generation--;
3987}
3988
3989static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
3990{
3991	hpa_t root_tdp = vcpu->arch.mmu->root.hpa;
3992
3993	/*
3994	 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly
3995	 * flush the NPT mappings via hypercall as flushing the ASID only
3996	 * affects virtual to physical mappings, it does not invalidate guest
3997	 * physical to host physical mappings.
3998	 */
3999	if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp))
4000		hyperv_flush_guest_mapping(root_tdp);
4001
4002	svm_flush_tlb_asid(vcpu);
4003}
4004
4005static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
4006{
4007	/*
4008	 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB
4009	 * flushes should be routed to hv_flush_remote_tlbs() without requesting
4010	 * a "regular" remote flush.  Reaching this point means either there's
4011	 * a KVM bug or a prior hv_flush_remote_tlbs() call failed, both of
4012	 * which might be fatal to the guest.  Yell, but try to recover.
4013	 */
4014	if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu)))
4015		hv_flush_remote_tlbs(vcpu->kvm);
4016
4017	svm_flush_tlb_asid(vcpu);
4018}
4019
4020static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
4021{
4022	struct vcpu_svm *svm = to_svm(vcpu);
4023
4024	invlpga(gva, svm->vmcb->control.asid);
4025}
4026
4027static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
4028{
4029	struct vcpu_svm *svm = to_svm(vcpu);
4030
4031	if (nested_svm_virtualize_tpr(vcpu))
4032		return;
4033
4034	if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
4035		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
4036		kvm_set_cr8(vcpu, cr8);
4037	}
4038}
4039
4040static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
4041{
4042	struct vcpu_svm *svm = to_svm(vcpu);
4043	u64 cr8;
4044
4045	if (nested_svm_virtualize_tpr(vcpu) ||
4046	    kvm_vcpu_apicv_active(vcpu))
4047		return;
4048
4049	cr8 = kvm_get_cr8(vcpu);
4050	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
4051	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
4052}
4053
4054static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector,
4055					int type)
4056{
4057	bool is_exception = (type == SVM_EXITINTINFO_TYPE_EXEPT);
4058	bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT);
4059	struct vcpu_svm *svm = to_svm(vcpu);
4060
4061	/*
4062	 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's
4063	 * associated with the original soft exception/interrupt.  next_rip is
4064	 * cleared on all exits that can occur while vectoring an event, so KVM
4065	 * needs to manually set next_rip for re-injection.  Unlike the !nrips
4066	 * case below, this needs to be done if and only if KVM is re-injecting
4067	 * the same event, i.e. if the event is a soft exception/interrupt,
4068	 * otherwise next_rip is unused on VMRUN.
4069	 */
4070	if (nrips && (is_soft || (is_exception && kvm_exception_is_soft(vector))) &&
4071	    kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase))
4072		svm->vmcb->control.next_rip = svm->soft_int_next_rip;
4073	/*
4074	 * If NRIPS isn't enabled, KVM must manually advance RIP prior to
4075	 * injecting the soft exception/interrupt.  That advancement needs to
4076	 * be unwound if vectoring didn't complete.  Note, the new event may
4077	 * not be the injected event, e.g. if KVM injected an INTn, the INTn
4078	 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will
4079	 * be the reported vectored event, but RIP still needs to be unwound.
4080	 */
4081	else if (!nrips && (is_soft || is_exception) &&
4082		 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase))
4083		kvm_rip_write(vcpu, svm->soft_int_old_rip);
4084}
4085
4086static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
4087{
4088	struct vcpu_svm *svm = to_svm(vcpu);
4089	u8 vector;
4090	int type;
4091	u32 exitintinfo = svm->vmcb->control.exit_int_info;
4092	bool nmi_l1_to_l2 = svm->nmi_l1_to_l2;
4093	bool soft_int_injected = svm->soft_int_injected;
4094
4095	svm->nmi_l1_to_l2 = false;
4096	svm->soft_int_injected = false;
4097
4098	/*
4099	 * If we've made progress since setting awaiting_iret_completion, we've
4100	 * executed an IRET and can allow NMI injection.
4101	 */
4102	if (svm->awaiting_iret_completion &&
4103	    kvm_rip_read(vcpu) != svm->nmi_iret_rip) {
4104		svm->awaiting_iret_completion = false;
4105		svm->nmi_masked = false;
4106		kvm_make_request(KVM_REQ_EVENT, vcpu);
4107	}
4108
4109	vcpu->arch.nmi_injected = false;
4110	kvm_clear_exception_queue(vcpu);
4111	kvm_clear_interrupt_queue(vcpu);
4112
4113	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
4114		return;
4115
4116	kvm_make_request(KVM_REQ_EVENT, vcpu);
4117
4118	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
4119	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
4120
4121	if (soft_int_injected)
4122		svm_complete_soft_interrupt(vcpu, vector, type);
4123
4124	switch (type) {
4125	case SVM_EXITINTINFO_TYPE_NMI:
4126		vcpu->arch.nmi_injected = true;
4127		svm->nmi_l1_to_l2 = nmi_l1_to_l2;
4128		break;
4129	case SVM_EXITINTINFO_TYPE_EXEPT:
4130		/*
4131		 * Never re-inject a #VC exception.
4132		 */
4133		if (vector == X86_TRAP_VC)
4134			break;
4135
4136		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
4137			u32 err = svm->vmcb->control.exit_int_info_err;
4138			kvm_requeue_exception_e(vcpu, vector, err);
4139
4140		} else
4141			kvm_requeue_exception(vcpu, vector);
4142		break;
4143	case SVM_EXITINTINFO_TYPE_INTR:
4144		kvm_queue_interrupt(vcpu, vector, false);
4145		break;
4146	case SVM_EXITINTINFO_TYPE_SOFT:
4147		kvm_queue_interrupt(vcpu, vector, true);
4148		break;
4149	default:
4150		break;
4151	}
4152
4153}
4154
4155static void svm_cancel_injection(struct kvm_vcpu *vcpu)
4156{
4157	struct vcpu_svm *svm = to_svm(vcpu);
4158	struct vmcb_control_area *control = &svm->vmcb->control;
4159
4160	control->exit_int_info = control->event_inj;
4161	control->exit_int_info_err = control->event_inj_err;
4162	control->event_inj = 0;
4163	svm_complete_interrupts(vcpu);
4164}
4165
4166static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu)
4167{
4168	if (to_kvm_sev_info(vcpu->kvm)->need_init)
4169		return -EINVAL;
4170
4171	return 1;
4172}
4173
4174static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
4175{
4176	struct vcpu_svm *svm = to_svm(vcpu);
4177
4178	if (is_guest_mode(vcpu))
4179		return EXIT_FASTPATH_NONE;
4180
4181	switch (svm->vmcb->control.exit_code) {
4182	case SVM_EXIT_MSR:
4183		if (!svm->vmcb->control.exit_info_1)
4184			break;
4185		return handle_fastpath_set_msr_irqoff(vcpu);
4186	case SVM_EXIT_HLT:
4187		return handle_fastpath_hlt(vcpu);
4188	default:
4189		break;
4190	}
4191
4192	return EXIT_FASTPATH_NONE;
4193}
4194
4195static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
4196{
4197	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
4198	struct vcpu_svm *svm = to_svm(vcpu);
4199
4200	guest_state_enter_irqoff();
4201
4202	/*
4203	 * Set RFLAGS.IF prior to VMRUN, as the host's RFLAGS.IF at the time of
4204	 * VMRUN controls whether or not physical IRQs are masked (KVM always
4205	 * runs with V_INTR_MASKING_MASK).  Toggle RFLAGS.IF here to avoid the
4206	 * temptation to do STI+VMRUN+CLI, as AMD CPUs bleed the STI shadow
4207	 * into guest state if delivery of an event during VMRUN triggers a
4208	 * #VMEXIT, and the guest_state transitions already tell lockdep that
4209	 * IRQs are being enabled/disabled.  Note!  GIF=0 for the entirety of
4210	 * this path, so IRQs aren't actually unmasked while running host code.
4211	 */
4212	raw_local_irq_enable();
4213
4214	amd_clear_divider();
4215
4216	if (sev_es_guest(vcpu->kvm))
4217		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted,
4218				      sev_es_host_save_area(sd));
4219	else
4220		__svm_vcpu_run(svm, spec_ctrl_intercepted);
4221
4222	raw_local_irq_disable();
4223
4224	guest_state_exit_irqoff();
4225}
4226
4227static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu,
4228					  bool force_immediate_exit)
4229{
4230	struct vcpu_svm *svm = to_svm(vcpu);
4231	bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL);
4232
4233	trace_kvm_entry(vcpu, force_immediate_exit);
4234
4235	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4236	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4237	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4238
4239	/*
4240	 * Disable singlestep if we're injecting an interrupt/exception.
4241	 * We don't want our modified rflags to be pushed on the stack where
4242	 * we might not be able to easily reset them if we disabled NMI
4243	 * singlestep later.
4244	 */
4245	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
4246		/*
4247		 * Event injection happens before external interrupts cause a
4248		 * vmexit and interrupts are disabled here, so smp_send_reschedule
4249		 * is enough to force an immediate vmexit.
4250		 */
4251		disable_nmi_singlestep(svm);
4252		force_immediate_exit = true;
4253	}
4254
4255	if (force_immediate_exit)
4256		smp_send_reschedule(vcpu->cpu);
 
4257
4258	pre_svm_run(vcpu);
4259
4260	sync_lapic_to_cr8(vcpu);
4261
4262	if (unlikely(svm->asid != svm->vmcb->control.asid)) {
4263		svm->vmcb->control.asid = svm->asid;
4264		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
4265	}
4266	svm->vmcb->save.cr2 = vcpu->arch.cr2;
4267
4268	svm_hv_update_vp_id(svm->vmcb, vcpu);
4269
4270	/*
4271	 * Run with all-zero DR6 unless needed, so that we can get the exact cause
4272	 * of a #DB.
4273	 */
4274	if (likely(!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)))
4275		svm_set_dr6(vcpu, DR6_ACTIVE_LOW);
 
 
4276
4277	clgi();
4278	kvm_load_guest_xsave_state(vcpu);
4279
4280	/*
4281	 * Hardware only context switches DEBUGCTL if LBR virtualization is
4282	 * enabled.  Manually load DEBUGCTL if necessary (and restore it after
4283	 * VM-Exit), as running with the host's DEBUGCTL can negatively affect
4284	 * guest state and can even be fatal, e.g. due to Bus Lock Detect.
4285	 */
4286	if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) &&
4287	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
4288		update_debugctlmsr(svm->vmcb->save.dbgctl);
4289
4290	kvm_wait_lapic_expire(vcpu);
4291
4292	/*
4293	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
4294	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
4295	 * is no need to worry about the conditional branch over the wrmsr
4296	 * being speculatively taken.
4297	 */
4298	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4299		x86_spec_ctrl_set_guest(svm->virt_spec_ctrl);
4300
4301	svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted);
4302
4303	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4304		x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
4305
4306	if (!sev_es_guest(vcpu->kvm)) {
4307		vcpu->arch.cr2 = svm->vmcb->save.cr2;
4308		vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
4309		vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
4310		vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
4311	}
4312	vcpu->arch.regs_dirty = 0;
4313
4314	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4315		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
4316
4317	if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) &&
4318	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
4319		update_debugctlmsr(vcpu->arch.host_debugctl);
4320
4321	kvm_load_host_xsave_state(vcpu);
4322	stgi();
4323
4324	/* Any pending NMI will happen here */
4325
4326	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4327		kvm_after_interrupt(vcpu);
4328
4329	sync_cr8_to_lapic(vcpu);
4330
4331	svm->next_rip = 0;
4332	if (is_guest_mode(vcpu)) {
4333		nested_sync_control_from_vmcb02(svm);
4334
4335		/* Track VMRUNs that have made past consistency checking */
4336		if (svm->nested.nested_run_pending &&
4337		    svm->vmcb->control.exit_code != SVM_EXIT_ERR)
4338                        ++vcpu->stat.nested_run;
4339
4340		svm->nested.nested_run_pending = 0;
4341	}
4342
4343	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4344	vmcb_mark_all_clean(svm->vmcb);
4345
4346	/* if exit due to PF check for async PF */
4347	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
4348		vcpu->arch.apf.host_apf_flags =
4349			kvm_read_and_reset_apf_flags();
4350
4351	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
4352
4353	/*
4354	 * We need to handle MC intercepts here before the vcpu has a chance to
4355	 * change the physical cpu
4356	 */
4357	if (unlikely(svm->vmcb->control.exit_code ==
4358		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
4359		svm_handle_mce(vcpu);
4360
4361	trace_kvm_exit(vcpu, KVM_ISA_SVM);
4362
4363	svm_complete_interrupts(vcpu);
4364
 
 
 
4365	return svm_exit_handlers_fastpath(vcpu);
4366}
4367
4368static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
4369			     int root_level)
4370{
4371	struct vcpu_svm *svm = to_svm(vcpu);
4372	unsigned long cr3;
4373
4374	if (npt_enabled) {
4375		svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
4376		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
4377
4378		hv_track_root_tdp(vcpu, root_hpa);
4379
4380		cr3 = vcpu->arch.cr3;
4381	} else if (root_level >= PT64_ROOT_4LEVEL) {
4382		cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
4383	} else {
4384		/* PCID in the guest should be impossible with a 32-bit MMU. */
4385		WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
4386		cr3 = root_hpa;
4387	}
4388
4389	svm->vmcb->save.cr3 = cr3;
4390	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
4391}
4392
4393static void
4394svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4395{
4396	/*
4397	 * Patch in the VMMCALL instruction:
4398	 */
4399	hypercall[0] = 0x0f;
4400	hypercall[1] = 0x01;
4401	hypercall[2] = 0xd9;
4402}
4403
4404/*
4405 * The kvm parameter can be NULL (module initialization, or invocation before
4406 * VM creation). Be sure to check the kvm parameter before using it.
4407 */
4408static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
4409{
4410	switch (index) {
4411	case MSR_IA32_MCG_EXT_CTL:
4412	case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR:
4413		return false;
4414	case MSR_IA32_SMBASE:
4415		if (!IS_ENABLED(CONFIG_KVM_SMM))
4416			return false;
4417		/* SEV-ES guests do not support SMM, so report false */
4418		if (kvm && sev_es_guest(kvm))
4419			return false;
4420		break;
4421	default:
4422		break;
4423	}
4424
4425	return true;
4426}
4427
4428static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4429{
4430	struct vcpu_svm *svm = to_svm(vcpu);
4431
4432	/*
4433	 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM
4434	 * can only disable all variants of by disallowing CR4.OSXSAVE from
4435	 * being set.  As a result, if the host has XSAVE and XSAVES, and the
4436	 * guest has XSAVE enabled, the guest can execute XSAVES without
4437	 * faulting.  Treat XSAVES as enabled in this case regardless of
4438	 * whether it's advertised to the guest so that KVM context switches
4439	 * XSS on VM-Enter/VM-Exit.  Failure to do so would effectively give
4440	 * the guest read/write access to the host's XSS.
4441	 */
4442	if (boot_cpu_has(X86_FEATURE_XSAVE) &&
4443	    boot_cpu_has(X86_FEATURE_XSAVES) &&
4444	    guest_cpuid_has(vcpu, X86_FEATURE_XSAVE))
4445		kvm_governed_feature_set(vcpu, X86_FEATURE_XSAVES);
4446
4447	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_NRIPS);
4448	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_TSCRATEMSR);
4449	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_LBRV);
4450
4451	/*
4452	 * Intercept VMLOAD if the vCPU model is Intel in order to emulate that
4453	 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing
4454	 * SVM on Intel is bonkers and extremely unlikely to work).
4455	 */
4456	if (!guest_cpuid_is_intel_compatible(vcpu))
4457		kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD);
4458
4459	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PAUSEFILTER);
4460	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PFTHRESHOLD);
4461	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VGIF);
4462	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VNMI);
4463
4464	svm_recalc_instruction_intercepts(vcpu, svm);
4465
4466	if (boot_cpu_has(X86_FEATURE_IBPB))
4467		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0,
4468				     !!guest_has_pred_cmd_msr(vcpu));
4469
4470	if (boot_cpu_has(X86_FEATURE_FLUSH_L1D))
4471		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0,
4472				     !!guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D));
4473
4474	if (sev_guest(vcpu->kvm))
4475		sev_vcpu_after_set_cpuid(svm);
4476
4477	init_vmcb_after_set_cpuid(vcpu);
4478}
4479
4480static bool svm_has_wbinvd_exit(void)
4481{
4482	return true;
4483}
4484
4485#define PRE_EX(exit)  { .exit_code = (exit), \
4486			.stage = X86_ICPT_PRE_EXCEPT, }
4487#define POST_EX(exit) { .exit_code = (exit), \
4488			.stage = X86_ICPT_POST_EXCEPT, }
4489#define POST_MEM(exit) { .exit_code = (exit), \
4490			.stage = X86_ICPT_POST_MEMACCESS, }
4491
4492static const struct __x86_intercept {
4493	u32 exit_code;
4494	enum x86_intercept_stage stage;
4495} x86_intercept_map[] = {
4496	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
4497	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
4498	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
4499	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
4500	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
4501	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
4502	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
4503	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
4504	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
4505	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
4506	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
4507	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
4508	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
4509	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
4510	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
4511	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
4512	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
4513	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
4514	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
4515	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
4516	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
4517	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
4518	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
4519	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
4520	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
4521	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
4522	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
4523	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
4524	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
4525	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
4526	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
4527	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
4528	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
4529	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
4530	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
4531	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
4532	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
4533	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
4534	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
4535	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
4536	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
4537	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
4538	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
4539	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
4540	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
4541	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
4542	[x86_intercept_xsetbv]		= PRE_EX(SVM_EXIT_XSETBV),
4543};
4544
4545#undef PRE_EX
4546#undef POST_EX
4547#undef POST_MEM
4548
4549static int svm_check_intercept(struct kvm_vcpu *vcpu,
4550			       struct x86_instruction_info *info,
4551			       enum x86_intercept_stage stage,
4552			       struct x86_exception *exception)
4553{
4554	struct vcpu_svm *svm = to_svm(vcpu);
4555	int vmexit, ret = X86EMUL_CONTINUE;
4556	struct __x86_intercept icpt_info;
4557	struct vmcb *vmcb = svm->vmcb;
4558
4559	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4560		goto out;
4561
4562	icpt_info = x86_intercept_map[info->intercept];
4563
4564	if (stage != icpt_info.stage)
4565		goto out;
4566
4567	switch (icpt_info.exit_code) {
4568	case SVM_EXIT_READ_CR0:
4569		if (info->intercept == x86_intercept_cr_read)
4570			icpt_info.exit_code += info->modrm_reg;
4571		break;
4572	case SVM_EXIT_WRITE_CR0: {
4573		unsigned long cr0, val;
4574
4575		if (info->intercept == x86_intercept_cr_write)
4576			icpt_info.exit_code += info->modrm_reg;
4577
4578		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4579		    info->intercept == x86_intercept_clts)
4580			break;
4581
4582		if (!(vmcb12_is_intercept(&svm->nested.ctl,
4583					INTERCEPT_SELECTIVE_CR0)))
4584			break;
4585
4586		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4587		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4588
4589		if (info->intercept == x86_intercept_lmsw) {
4590			cr0 &= 0xfUL;
4591			val &= 0xfUL;
4592			/* lmsw can't clear PE - catch this here */
4593			if (cr0 & X86_CR0_PE)
4594				val |= X86_CR0_PE;
4595		}
4596
4597		if (cr0 ^ val)
4598			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4599
4600		break;
4601	}
4602	case SVM_EXIT_READ_DR0:
4603	case SVM_EXIT_WRITE_DR0:
4604		icpt_info.exit_code += info->modrm_reg;
4605		break;
4606	case SVM_EXIT_MSR:
4607		if (info->intercept == x86_intercept_wrmsr)
4608			vmcb->control.exit_info_1 = 1;
4609		else
4610			vmcb->control.exit_info_1 = 0;
4611		break;
4612	case SVM_EXIT_PAUSE:
4613		/*
4614		 * We get this for NOP only, but pause
4615		 * is rep not, check this here
4616		 */
4617		if (info->rep_prefix != REPE_PREFIX)
4618			goto out;
4619		break;
4620	case SVM_EXIT_IOIO: {
4621		u64 exit_info;
4622		u32 bytes;
4623
4624		if (info->intercept == x86_intercept_in ||
4625		    info->intercept == x86_intercept_ins) {
4626			exit_info = ((info->src_val & 0xffff) << 16) |
4627				SVM_IOIO_TYPE_MASK;
4628			bytes = info->dst_bytes;
4629		} else {
4630			exit_info = (info->dst_val & 0xffff) << 16;
4631			bytes = info->src_bytes;
4632		}
4633
4634		if (info->intercept == x86_intercept_outs ||
4635		    info->intercept == x86_intercept_ins)
4636			exit_info |= SVM_IOIO_STR_MASK;
4637
4638		if (info->rep_prefix)
4639			exit_info |= SVM_IOIO_REP_MASK;
4640
4641		bytes = min(bytes, 4u);
4642
4643		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4644
4645		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4646
4647		vmcb->control.exit_info_1 = exit_info;
4648		vmcb->control.exit_info_2 = info->next_rip;
4649
4650		break;
4651	}
4652	default:
4653		break;
4654	}
4655
4656	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4657	if (static_cpu_has(X86_FEATURE_NRIPS))
4658		vmcb->control.next_rip  = info->next_rip;
4659	vmcb->control.exit_code = icpt_info.exit_code;
4660	vmexit = nested_svm_exit_handled(svm);
4661
4662	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4663					   : X86EMUL_CONTINUE;
4664
4665out:
4666	return ret;
4667}
4668
4669static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4670{
4671	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR)
4672		vcpu->arch.at_instruction_boundary = true;
4673}
4674
 
 
 
 
 
 
4675static void svm_setup_mce(struct kvm_vcpu *vcpu)
4676{
4677	/* [63:9] are reserved. */
4678	vcpu->arch.mcg_cap &= 0x1ff;
4679}
4680
4681#ifdef CONFIG_KVM_SMM
4682bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4683{
4684	struct vcpu_svm *svm = to_svm(vcpu);
4685
4686	/* Per APM Vol.2 15.22.2 "Response to SMI" */
4687	if (!gif_set(svm))
4688		return true;
4689
4690	return is_smm(vcpu);
4691}
4692
4693static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4694{
4695	struct vcpu_svm *svm = to_svm(vcpu);
4696	if (svm->nested.nested_run_pending)
4697		return -EBUSY;
4698
4699	if (svm_smi_blocked(vcpu))
4700		return 0;
4701
4702	/* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4703	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4704		return -EBUSY;
4705
4706	return 1;
4707}
4708
4709static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
4710{
4711	struct vcpu_svm *svm = to_svm(vcpu);
4712	struct kvm_host_map map_save;
4713	int ret;
4714
4715	if (!is_guest_mode(vcpu))
4716		return 0;
4717
4718	/*
4719	 * 32-bit SMRAM format doesn't preserve EFER and SVM state.  Userspace is
4720	 * responsible for ensuring nested SVM and SMIs are mutually exclusive.
4721	 */
4722
4723	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4724		return 1;
4725
4726	smram->smram64.svm_guest_flag = 1;
4727	smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa;
4728
4729	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4730	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4731	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4732
4733	ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
4734	if (ret)
4735		return ret;
4736
4737	/*
4738	 * KVM uses VMCB01 to store L1 host state while L2 runs but
4739	 * VMCB01 is going to be used during SMM and thus the state will
4740	 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4741	 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4742	 * format of the area is identical to guest save area offsetted
4743	 * by 0x400 (matches the offset of 'struct vmcb_save_area'
4744	 * within 'struct vmcb'). Note: HSAVE area may also be used by
4745	 * L1 hypervisor to save additional host context (e.g. KVM does
4746	 * that, see svm_prepare_switch_to_guest()) which must be
4747	 * preserved.
4748	 */
4749	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4750		return 1;
4751
4752	BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4753
4754	svm_copy_vmrun_state(map_save.hva + 0x400,
4755			     &svm->vmcb01.ptr->save);
4756
4757	kvm_vcpu_unmap(vcpu, &map_save);
4758	return 0;
4759}
4760
4761static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
4762{
4763	struct vcpu_svm *svm = to_svm(vcpu);
4764	struct kvm_host_map map, map_save;
4765	struct vmcb *vmcb12;
4766	int ret;
4767
4768	const struct kvm_smram_state_64 *smram64 = &smram->smram64;
4769
4770	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4771		return 0;
4772
4773	/* Non-zero if SMI arrived while vCPU was in guest mode. */
4774	if (!smram64->svm_guest_flag)
4775		return 0;
4776
4777	if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4778		return 1;
4779
4780	if (!(smram64->efer & EFER_SVME))
4781		return 1;
4782
4783	if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map))
4784		return 1;
4785
4786	ret = 1;
4787	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4788		goto unmap_map;
4789
4790	if (svm_allocate_nested(svm))
4791		goto unmap_save;
4792
4793	/*
4794	 * Restore L1 host state from L1 HSAVE area as VMCB01 was
4795	 * used during SMM (see svm_enter_smm())
4796	 */
4797
4798	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4799
4800	/*
4801	 * Enter the nested guest now
4802	 */
4803
4804	vmcb_mark_all_dirty(svm->vmcb01.ptr);
4805
4806	vmcb12 = map.hva;
4807	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
4808	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
4809	ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false);
4810
4811	if (ret)
4812		goto unmap_save;
4813
4814	svm->nested.nested_run_pending = 1;
4815
4816unmap_save:
4817	kvm_vcpu_unmap(vcpu, &map_save);
4818unmap_map:
4819	kvm_vcpu_unmap(vcpu, &map);
4820	return ret;
4821}
4822
4823static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4824{
4825	struct vcpu_svm *svm = to_svm(vcpu);
4826
4827	if (!gif_set(svm)) {
4828		if (vgif)
4829			svm_set_intercept(svm, INTERCEPT_STGI);
4830		/* STGI will cause a vm exit */
4831	} else {
4832		/* We must be in SMM; RSM will cause a vmexit anyway.  */
4833	}
4834}
4835#endif
4836
4837static int svm_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
4838					 void *insn, int insn_len)
4839{
4840	bool smep, smap, is_user;
4841	u64 error_code;
4842
4843	/* Emulation is always possible when KVM has access to all guest state. */
4844	if (!sev_guest(vcpu->kvm))
4845		return X86EMUL_CONTINUE;
4846
4847	/* #UD and #GP should never be intercepted for SEV guests. */
4848	WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD |
4849				  EMULTYPE_TRAP_UD_FORCED |
4850				  EMULTYPE_VMWARE_GP));
4851
4852	/*
4853	 * Emulation is impossible for SEV-ES guests as KVM doesn't have access
4854	 * to guest register state.
4855	 */
4856	if (sev_es_guest(vcpu->kvm))
4857		return X86EMUL_RETRY_INSTR;
4858
4859	/*
4860	 * Emulation is possible if the instruction is already decoded, e.g.
4861	 * when completing I/O after returning from userspace.
4862	 */
4863	if (emul_type & EMULTYPE_NO_DECODE)
4864		return X86EMUL_CONTINUE;
4865
4866	/*
4867	 * Emulation is possible for SEV guests if and only if a prefilled
4868	 * buffer containing the bytes of the intercepted instruction is
4869	 * available. SEV guest memory is encrypted with a guest specific key
4870	 * and cannot be decrypted by KVM, i.e. KVM would read ciphertext and
4871	 * decode garbage.
4872	 *
4873	 * If KVM is NOT trying to simply skip an instruction, inject #UD if
4874	 * KVM reached this point without an instruction buffer.  In practice,
4875	 * this path should never be hit by a well-behaved guest, e.g. KVM
4876	 * doesn't intercept #UD or #GP for SEV guests, but this path is still
4877	 * theoretically reachable, e.g. via unaccelerated fault-like AVIC
4878	 * access, and needs to be handled by KVM to avoid putting the guest
4879	 * into an infinite loop.   Injecting #UD is somewhat arbitrary, but
4880	 * its the least awful option given lack of insight into the guest.
4881	 *
4882	 * If KVM is trying to skip an instruction, simply resume the guest.
4883	 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM
4884	 * will attempt to re-inject the INT3/INTO and skip the instruction.
4885	 * In that scenario, retrying the INT3/INTO and hoping the guest will
4886	 * make forward progress is the only option that has a chance of
4887	 * success (and in practice it will work the vast majority of the time).
4888	 */
4889	if (unlikely(!insn)) {
4890		if (emul_type & EMULTYPE_SKIP)
4891			return X86EMUL_UNHANDLEABLE;
4892
4893		kvm_queue_exception(vcpu, UD_VECTOR);
4894		return X86EMUL_PROPAGATE_FAULT;
4895	}
4896
4897	/*
4898	 * Emulate for SEV guests if the insn buffer is not empty.  The buffer
4899	 * will be empty if the DecodeAssist microcode cannot fetch bytes for
4900	 * the faulting instruction because the code fetch itself faulted, e.g.
4901	 * the guest attempted to fetch from emulated MMIO or a guest page
4902	 * table used to translate CS:RIP resides in emulated MMIO.
4903	 */
4904	if (likely(insn_len))
4905		return X86EMUL_CONTINUE;
4906
4907	/*
4908	 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4909	 *
4910	 * Errata:
4911	 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
4912	 * possible that CPU microcode implementing DecodeAssist will fail to
4913	 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
4914	 * be '0'.  This happens because microcode reads CS:RIP using a _data_
4915	 * loap uop with CPL=0 privileges.  If the load hits a SMAP #PF, ucode
4916	 * gives up and does not fill the instruction bytes buffer.
4917	 *
4918	 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
4919	 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
4920	 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
4921	 * GuestIntrBytes field of the VMCB.
4922	 *
4923	 * This does _not_ mean that the erratum has been encountered, as the
4924	 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
4925	 * #PF, e.g. if the guest attempt to execute from emulated MMIO and
4926	 * encountered a reserved/not-present #PF.
4927	 *
4928	 * To hit the erratum, the following conditions must be true:
4929	 *    1. CR4.SMAP=1 (obviously).
4930	 *    2. CR4.SMEP=0 || CPL=3.  If SMEP=1 and CPL<3, the erratum cannot
4931	 *       have been hit as the guest would have encountered a SMEP
4932	 *       violation #PF, not a #NPF.
4933	 *    3. The #NPF is not due to a code fetch, in which case failure to
4934	 *       retrieve the instruction bytes is legitimate (see abvoe).
4935	 *
4936	 * In addition, don't apply the erratum workaround if the #NPF occurred
4937	 * while translating guest page tables (see below).
4938	 */
4939	error_code = to_svm(vcpu)->vmcb->control.exit_info_1;
4940	if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK))
4941		goto resume_guest;
4942
4943	smep = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMEP);
4944	smap = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMAP);
4945	is_user = svm_get_cpl(vcpu) == 3;
4946	if (smap && (!smep || is_user)) {
4947		pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n");
4948
4949		/*
4950		 * If the fault occurred in userspace, arbitrarily inject #GP
4951		 * to avoid killing the guest and to hopefully avoid confusing
4952		 * the guest kernel too much, e.g. injecting #PF would not be
4953		 * coherent with respect to the guest's page tables.  Request
4954		 * triple fault if the fault occurred in the kernel as there's
4955		 * no fault that KVM can inject without confusing the guest.
4956		 * In practice, the triple fault is moot as no sane SEV kernel
4957		 * will execute from user memory while also running with SMAP=1.
4958		 */
4959		if (is_user)
4960			kvm_inject_gp(vcpu, 0);
4961		else
4962			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4963		return X86EMUL_PROPAGATE_FAULT;
4964	}
4965
4966resume_guest:
4967	/*
4968	 * If the erratum was not hit, simply resume the guest and let it fault
4969	 * again.  While awful, e.g. the vCPU may get stuck in an infinite loop
4970	 * if the fault is at CPL=0, it's the lesser of all evils.  Exiting to
4971	 * userspace will kill the guest, and letting the emulator read garbage
4972	 * will yield random behavior and potentially corrupt the guest.
4973	 *
4974	 * Simply resuming the guest is technically not a violation of the SEV
4975	 * architecture.  AMD's APM states that all code fetches and page table
4976	 * accesses for SEV guest are encrypted, regardless of the C-Bit.  The
4977	 * APM also states that encrypted accesses to MMIO are "ignored", but
4978	 * doesn't explicitly define "ignored", i.e. doing nothing and letting
4979	 * the guest spin is technically "ignoring" the access.
4980	 */
4981	return X86EMUL_RETRY_INSTR;
4982}
4983
4984static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4985{
4986	struct vcpu_svm *svm = to_svm(vcpu);
4987
4988	return !gif_set(svm);
4989}
4990
4991static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4992{
4993	if (!sev_es_guest(vcpu->kvm))
4994		return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
4995
4996	sev_vcpu_deliver_sipi_vector(vcpu, vector);
4997}
4998
4999static void svm_vm_destroy(struct kvm *kvm)
5000{
5001	avic_vm_destroy(kvm);
5002	sev_vm_destroy(kvm);
5003}
5004
5005static int svm_vm_init(struct kvm *kvm)
5006{
5007	int type = kvm->arch.vm_type;
5008
5009	if (type != KVM_X86_DEFAULT_VM &&
5010	    type != KVM_X86_SW_PROTECTED_VM) {
5011		kvm->arch.has_protected_state =
5012			(type == KVM_X86_SEV_ES_VM || type == KVM_X86_SNP_VM);
5013		to_kvm_sev_info(kvm)->need_init = true;
5014
5015		kvm->arch.has_private_mem = (type == KVM_X86_SNP_VM);
5016		kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem;
5017	}
5018
5019	if (!pause_filter_count || !pause_filter_thresh)
5020		kvm->arch.pause_in_guest = true;
5021
5022	if (enable_apicv) {
5023		int ret = avic_vm_init(kvm);
5024		if (ret)
5025			return ret;
5026	}
5027
5028	return 0;
5029}
5030
5031static void *svm_alloc_apic_backing_page(struct kvm_vcpu *vcpu)
5032{
5033	struct page *page = snp_safe_alloc_page();
5034
5035	if (!page)
5036		return NULL;
5037
5038	return page_address(page);
5039}
5040
5041static struct kvm_x86_ops svm_x86_ops __initdata = {
5042	.name = KBUILD_MODNAME,
5043
5044	.check_processor_compatibility = svm_check_processor_compat,
5045
5046	.hardware_unsetup = svm_hardware_unsetup,
5047	.enable_virtualization_cpu = svm_enable_virtualization_cpu,
5048	.disable_virtualization_cpu = svm_disable_virtualization_cpu,
5049	.emergency_disable_virtualization_cpu = svm_emergency_disable_virtualization_cpu,
5050	.has_emulated_msr = svm_has_emulated_msr,
5051
5052	.vcpu_create = svm_vcpu_create,
5053	.vcpu_free = svm_vcpu_free,
5054	.vcpu_reset = svm_vcpu_reset,
5055
5056	.vm_size = sizeof(struct kvm_svm),
5057	.vm_init = svm_vm_init,
5058	.vm_destroy = svm_vm_destroy,
5059
5060	.prepare_switch_to_guest = svm_prepare_switch_to_guest,
5061	.vcpu_load = svm_vcpu_load,
5062	.vcpu_put = svm_vcpu_put,
5063	.vcpu_blocking = avic_vcpu_blocking,
5064	.vcpu_unblocking = avic_vcpu_unblocking,
5065
5066	.update_exception_bitmap = svm_update_exception_bitmap,
5067	.get_feature_msr = svm_get_feature_msr,
5068	.get_msr = svm_get_msr,
5069	.set_msr = svm_set_msr,
5070	.get_segment_base = svm_get_segment_base,
5071	.get_segment = svm_get_segment,
5072	.set_segment = svm_set_segment,
5073	.get_cpl = svm_get_cpl,
5074	.get_cpl_no_cache = svm_get_cpl,
5075	.get_cs_db_l_bits = svm_get_cs_db_l_bits,
5076	.is_valid_cr0 = svm_is_valid_cr0,
5077	.set_cr0 = svm_set_cr0,
5078	.post_set_cr3 = sev_post_set_cr3,
5079	.is_valid_cr4 = svm_is_valid_cr4,
5080	.set_cr4 = svm_set_cr4,
5081	.set_efer = svm_set_efer,
5082	.get_idt = svm_get_idt,
5083	.set_idt = svm_set_idt,
5084	.get_gdt = svm_get_gdt,
5085	.set_gdt = svm_set_gdt,
5086	.set_dr6 = svm_set_dr6,
5087	.set_dr7 = svm_set_dr7,
5088	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
5089	.cache_reg = svm_cache_reg,
5090	.get_rflags = svm_get_rflags,
5091	.set_rflags = svm_set_rflags,
5092	.get_if_flag = svm_get_if_flag,
5093
5094	.flush_tlb_all = svm_flush_tlb_all,
5095	.flush_tlb_current = svm_flush_tlb_current,
5096	.flush_tlb_gva = svm_flush_tlb_gva,
5097	.flush_tlb_guest = svm_flush_tlb_asid,
5098
5099	.vcpu_pre_run = svm_vcpu_pre_run,
5100	.vcpu_run = svm_vcpu_run,
5101	.handle_exit = svm_handle_exit,
5102	.skip_emulated_instruction = svm_skip_emulated_instruction,
5103	.update_emulated_instruction = NULL,
5104	.set_interrupt_shadow = svm_set_interrupt_shadow,
5105	.get_interrupt_shadow = svm_get_interrupt_shadow,
5106	.patch_hypercall = svm_patch_hypercall,
5107	.inject_irq = svm_inject_irq,
5108	.inject_nmi = svm_inject_nmi,
5109	.is_vnmi_pending = svm_is_vnmi_pending,
5110	.set_vnmi_pending = svm_set_vnmi_pending,
5111	.inject_exception = svm_inject_exception,
5112	.cancel_injection = svm_cancel_injection,
5113	.interrupt_allowed = svm_interrupt_allowed,
5114	.nmi_allowed = svm_nmi_allowed,
5115	.get_nmi_mask = svm_get_nmi_mask,
5116	.set_nmi_mask = svm_set_nmi_mask,
5117	.enable_nmi_window = svm_enable_nmi_window,
5118	.enable_irq_window = svm_enable_irq_window,
5119	.update_cr8_intercept = svm_update_cr8_intercept,
5120
5121	.x2apic_icr_is_split = true,
5122	.set_virtual_apic_mode = avic_refresh_virtual_apic_mode,
5123	.refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl,
5124	.apicv_post_state_restore = avic_apicv_post_state_restore,
5125	.required_apicv_inhibits = AVIC_REQUIRED_APICV_INHIBITS,
5126
5127	.get_exit_info = svm_get_exit_info,
5128
5129	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
5130
5131	.has_wbinvd_exit = svm_has_wbinvd_exit,
5132
5133	.get_l2_tsc_offset = svm_get_l2_tsc_offset,
5134	.get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
5135	.write_tsc_offset = svm_write_tsc_offset,
5136	.write_tsc_multiplier = svm_write_tsc_multiplier,
5137
5138	.load_mmu_pgd = svm_load_mmu_pgd,
5139
5140	.check_intercept = svm_check_intercept,
5141	.handle_exit_irqoff = svm_handle_exit_irqoff,
5142
 
 
 
 
5143	.nested_ops = &svm_nested_ops,
5144
5145	.deliver_interrupt = svm_deliver_interrupt,
5146	.pi_update_irte = avic_pi_update_irte,
5147	.setup_mce = svm_setup_mce,
5148
5149#ifdef CONFIG_KVM_SMM
5150	.smi_allowed = svm_smi_allowed,
5151	.enter_smm = svm_enter_smm,
5152	.leave_smm = svm_leave_smm,
5153	.enable_smi_window = svm_enable_smi_window,
5154#endif
5155
5156#ifdef CONFIG_KVM_AMD_SEV
5157	.dev_get_attr = sev_dev_get_attr,
5158	.mem_enc_ioctl = sev_mem_enc_ioctl,
5159	.mem_enc_register_region = sev_mem_enc_register_region,
5160	.mem_enc_unregister_region = sev_mem_enc_unregister_region,
5161	.guest_memory_reclaimed = sev_guest_memory_reclaimed,
5162
5163	.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
5164	.vm_move_enc_context_from = sev_vm_move_enc_context_from,
5165#endif
5166	.check_emulate_instruction = svm_check_emulate_instruction,
5167
5168	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
5169
5170	.msr_filter_changed = svm_msr_filter_changed,
5171	.complete_emulated_msr = svm_complete_emulated_msr,
5172
5173	.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
5174	.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
5175	.alloc_apic_backing_page = svm_alloc_apic_backing_page,
5176
5177	.gmem_prepare = sev_gmem_prepare,
5178	.gmem_invalidate = sev_gmem_invalidate,
5179	.private_max_mapping_level = sev_private_max_mapping_level,
5180};
5181
5182/*
5183 * The default MMIO mask is a single bit (excluding the present bit),
5184 * which could conflict with the memory encryption bit. Check for
5185 * memory encryption support and override the default MMIO mask if
5186 * memory encryption is enabled.
5187 */
5188static __init void svm_adjust_mmio_mask(void)
5189{
5190	unsigned int enc_bit, mask_bit;
5191	u64 msr, mask;
5192
5193	/* If there is no memory encryption support, use existing mask */
5194	if (cpuid_eax(0x80000000) < 0x8000001f)
5195		return;
5196
5197	/* If memory encryption is not enabled, use existing mask */
5198	rdmsrl(MSR_AMD64_SYSCFG, msr);
5199	if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
5200		return;
5201
5202	enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
5203	mask_bit = boot_cpu_data.x86_phys_bits;
5204
5205	/* Increment the mask bit if it is the same as the encryption bit */
5206	if (enc_bit == mask_bit)
5207		mask_bit++;
5208
5209	/*
5210	 * If the mask bit location is below 52, then some bits above the
5211	 * physical addressing limit will always be reserved, so use the
5212	 * rsvd_bits() function to generate the mask. This mask, along with
5213	 * the present bit, will be used to generate a page fault with
5214	 * PFER.RSV = 1.
5215	 *
5216	 * If the mask bit location is 52 (or above), then clear the mask.
5217	 */
5218	mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
5219
5220	kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
5221}
5222
5223static __init void svm_set_cpu_caps(void)
5224{
5225	kvm_set_cpu_caps();
5226
5227	kvm_caps.supported_perf_cap = 0;
5228	kvm_caps.supported_xss = 0;
5229
5230	/* CPUID 0x80000001 and 0x8000000A (SVM features) */
5231	if (nested) {
5232		kvm_cpu_cap_set(X86_FEATURE_SVM);
5233		kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN);
5234
5235		/*
5236		 * KVM currently flushes TLBs on *every* nested SVM transition,
5237		 * and so for all intents and purposes KVM supports flushing by
5238		 * ASID, i.e. KVM is guaranteed to honor every L1 ASID flush.
5239		 */
5240		kvm_cpu_cap_set(X86_FEATURE_FLUSHBYASID);
5241
5242		if (nrips)
5243			kvm_cpu_cap_set(X86_FEATURE_NRIPS);
5244
5245		if (npt_enabled)
5246			kvm_cpu_cap_set(X86_FEATURE_NPT);
5247
5248		if (tsc_scaling)
5249			kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR);
5250
5251		if (vls)
5252			kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD);
5253		if (lbrv)
5254			kvm_cpu_cap_set(X86_FEATURE_LBRV);
5255
5256		if (boot_cpu_has(X86_FEATURE_PAUSEFILTER))
5257			kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER);
5258
5259		if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD))
5260			kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD);
5261
5262		if (vgif)
5263			kvm_cpu_cap_set(X86_FEATURE_VGIF);
5264
5265		if (vnmi)
5266			kvm_cpu_cap_set(X86_FEATURE_VNMI);
5267
5268		/* Nested VM can receive #VMEXIT instead of triggering #GP */
5269		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
5270	}
5271
5272	/* CPUID 0x80000008 */
5273	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5274	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
5275		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
5276
5277	if (enable_pmu) {
5278		/*
5279		 * Enumerate support for PERFCTR_CORE if and only if KVM has
5280		 * access to enough counters to virtualize "core" support,
5281		 * otherwise limit vPMU support to the legacy number of counters.
5282		 */
5283		if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE)
5284			kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS,
5285							  kvm_pmu_cap.num_counters_gp);
5286		else
5287			kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE);
5288
5289		if (kvm_pmu_cap.version != 2 ||
5290		    !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE))
5291			kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2);
5292	}
5293
5294	/* CPUID 0x8000001F (SME/SEV features) */
5295	sev_set_cpu_caps();
5296
5297	/* Don't advertise Bus Lock Detect to guest if SVM support is absent */
5298	kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT);
5299}
5300
5301static __init int svm_hardware_setup(void)
5302{
5303	int cpu;
5304	struct page *iopm_pages;
5305	void *iopm_va;
5306	int r;
5307	unsigned int order = get_order(IOPM_SIZE);
5308
5309	/*
5310	 * NX is required for shadow paging and for NPT if the NX huge pages
5311	 * mitigation is enabled.
5312	 */
5313	if (!boot_cpu_has(X86_FEATURE_NX)) {
5314		pr_err_ratelimited("NX (Execute Disable) not supported\n");
5315		return -EOPNOTSUPP;
5316	}
5317	kvm_enable_efer_bits(EFER_NX);
5318
5319	iopm_pages = alloc_pages(GFP_KERNEL, order);
5320
5321	if (!iopm_pages)
5322		return -ENOMEM;
5323
5324	iopm_va = page_address(iopm_pages);
5325	memset(iopm_va, 0xff, PAGE_SIZE * (1 << order));
5326	iopm_base = __sme_page_pa(iopm_pages);
5327
5328	init_msrpm_offsets();
5329
5330	kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
5331				     XFEATURE_MASK_BNDCSR);
5332
5333	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
5334		kvm_enable_efer_bits(EFER_FFXSR);
5335
5336	if (tsc_scaling) {
5337		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
5338			tsc_scaling = false;
5339		} else {
5340			pr_info("TSC scaling supported\n");
5341			kvm_caps.has_tsc_control = true;
5342		}
5343	}
5344	kvm_caps.max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX;
5345	kvm_caps.tsc_scaling_ratio_frac_bits = 32;
5346
5347	tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
5348
5349	if (boot_cpu_has(X86_FEATURE_AUTOIBRS))
5350		kvm_enable_efer_bits(EFER_AUTOIBRS);
5351
5352	/* Check for pause filtering support */
5353	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
5354		pause_filter_count = 0;
5355		pause_filter_thresh = 0;
5356	} else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
5357		pause_filter_thresh = 0;
5358	}
5359
5360	if (nested) {
5361		pr_info("Nested Virtualization enabled\n");
5362		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
5363	}
5364
5365	/*
5366	 * KVM's MMU doesn't support using 2-level paging for itself, and thus
5367	 * NPT isn't supported if the host is using 2-level paging since host
5368	 * CR4 is unchanged on VMRUN.
5369	 */
5370	if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
5371		npt_enabled = false;
5372
5373	if (!boot_cpu_has(X86_FEATURE_NPT))
5374		npt_enabled = false;
5375
5376	/* Force VM NPT level equal to the host's paging level */
5377	kvm_configure_mmu(npt_enabled, get_npt_level(),
5378			  get_npt_level(), PG_LEVEL_1G);
5379	pr_info("Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
5380
5381	/* Setup shadow_me_value and shadow_me_mask */
5382	kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask);
5383
5384	svm_adjust_mmio_mask();
5385
5386	nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS);
5387
5388	if (lbrv) {
5389		if (!boot_cpu_has(X86_FEATURE_LBRV))
5390			lbrv = false;
5391		else
5392			pr_info("LBR virtualization supported\n");
5393	}
5394	/*
5395	 * Note, SEV setup consumes npt_enabled and enable_mmio_caching (which
5396	 * may be modified by svm_adjust_mmio_mask()), as well as nrips.
5397	 */
5398	sev_hardware_setup();
5399
5400	svm_hv_hardware_setup();
5401
5402	for_each_possible_cpu(cpu) {
5403		r = svm_cpu_init(cpu);
5404		if (r)
5405			goto err;
5406	}
5407
5408	enable_apicv = avic = avic && avic_hardware_setup();
5409
5410	if (!enable_apicv) {
5411		svm_x86_ops.vcpu_blocking = NULL;
5412		svm_x86_ops.vcpu_unblocking = NULL;
5413		svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL;
5414	} else if (!x2avic_enabled) {
5415		svm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization = true;
5416	}
5417
5418	if (vls) {
5419		if (!npt_enabled ||
5420		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
5421		    !IS_ENABLED(CONFIG_X86_64)) {
5422			vls = false;
5423		} else {
5424			pr_info("Virtual VMLOAD VMSAVE supported\n");
5425		}
5426	}
5427
5428	if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
5429		svm_gp_erratum_intercept = false;
5430
5431	if (vgif) {
5432		if (!boot_cpu_has(X86_FEATURE_VGIF))
5433			vgif = false;
5434		else
5435			pr_info("Virtual GIF supported\n");
5436	}
5437
5438	vnmi = vgif && vnmi && boot_cpu_has(X86_FEATURE_VNMI);
5439	if (vnmi)
5440		pr_info("Virtual NMI enabled\n");
5441
5442	if (!vnmi) {
5443		svm_x86_ops.is_vnmi_pending = NULL;
5444		svm_x86_ops.set_vnmi_pending = NULL;
5445	}
5446
 
 
 
 
 
 
 
 
5447	if (!enable_pmu)
5448		pr_info("PMU virtualization is disabled\n");
5449
5450	svm_set_cpu_caps();
5451
5452	/*
5453	 * It seems that on AMD processors PTE's accessed bit is
5454	 * being set by the CPU hardware before the NPF vmexit.
5455	 * This is not expected behaviour and our tests fail because
5456	 * of it.
5457	 * A workaround here is to disable support for
5458	 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
5459	 * In this case userspace can know if there is support using
5460	 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
5461	 * it
5462	 * If future AMD CPU models change the behaviour described above,
5463	 * this variable can be changed accordingly
5464	 */
5465	allow_smaller_maxphyaddr = !npt_enabled;
5466
5467	return 0;
5468
5469err:
5470	svm_hardware_unsetup();
5471	return r;
5472}
5473
5474
5475static struct kvm_x86_init_ops svm_init_ops __initdata = {
5476	.hardware_setup = svm_hardware_setup,
5477
5478	.runtime_ops = &svm_x86_ops,
5479	.pmu_ops = &amd_pmu_ops,
5480};
5481
5482static void __svm_exit(void)
5483{
5484	kvm_x86_vendor_exit();
 
 
5485}
5486
5487static int __init svm_init(void)
5488{
5489	int r;
5490
5491	__unused_size_checks();
5492
5493	if (!kvm_is_svm_supported())
5494		return -EOPNOTSUPP;
5495
5496	r = kvm_x86_vendor_init(&svm_init_ops);
5497	if (r)
5498		return r;
 
 
5499
5500	/*
5501	 * Common KVM initialization _must_ come last, after this, /dev/kvm is
5502	 * exposed to userspace!
5503	 */
5504	r = kvm_init(sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm),
5505		     THIS_MODULE);
5506	if (r)
5507		goto err_kvm_init;
5508
5509	return 0;
5510
5511err_kvm_init:
5512	__svm_exit();
5513	return r;
5514}
5515
5516static void __exit svm_exit(void)
5517{
5518	kvm_exit();
5519	__svm_exit();
5520}
5521
5522module_init(svm_init)
5523module_exit(svm_exit)