1// SPDX-License-Identifier: GPL-2.0
   2#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   3
   4#include <linux/objtool.h>
   5#include <linux/percpu.h>
   6
   7#include <asm/debugreg.h>
   8#include <asm/mmu_context.h>
   9
  10#include "x86.h"
  11#include "cpuid.h"
  12#include "hyperv.h"
  13#include "mmu.h"
  14#include "nested.h"
  15#include "pmu.h"
  16#include "posted_intr.h"
  17#include "sgx.h"
  18#include "trace.h"
  19#include "vmx.h"
 
  20#include "smm.h"
  21
  22static bool __read_mostly enable_shadow_vmcs = 1;
  23module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
  24
  25static bool __read_mostly nested_early_check = 0;
  26module_param(nested_early_check, bool, S_IRUGO);
  27
  28#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
  29
  30/*
  31 * Hyper-V requires all of these, so mark them as supported even though
  32 * they are just treated the same as all-context.
  33 */
  34#define VMX_VPID_EXTENT_SUPPORTED_MASK		\
  35	(VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |	\
  36	VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |	\
  37	VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |	\
  38	VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
  39
  40#define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
  41
  42enum {
  43	VMX_VMREAD_BITMAP,
  44	VMX_VMWRITE_BITMAP,
  45	VMX_BITMAP_NR
  46};
  47static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
  48
  49#define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
  50#define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
  51
  52struct shadow_vmcs_field {
  53	u16	encoding;
  54	u16	offset;
  55};
  56static struct shadow_vmcs_field shadow_read_only_fields[] = {
  57#define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) },
  58#include "vmcs_shadow_fields.h"
  59};
  60static int max_shadow_read_only_fields =
  61	ARRAY_SIZE(shadow_read_only_fields);
  62
  63static struct shadow_vmcs_field shadow_read_write_fields[] = {
  64#define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) },
  65#include "vmcs_shadow_fields.h"
  66};
  67static int max_shadow_read_write_fields =
  68	ARRAY_SIZE(shadow_read_write_fields);
  69
  70static void init_vmcs_shadow_fields(void)
  71{
  72	int i, j;
  73
  74	memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
  75	memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
  76
  77	for (i = j = 0; i < max_shadow_read_only_fields; i++) {
  78		struct shadow_vmcs_field entry = shadow_read_only_fields[i];
  79		u16 field = entry.encoding;
  80
  81		if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
  82		    (i + 1 == max_shadow_read_only_fields ||
  83		     shadow_read_only_fields[i + 1].encoding != field + 1))
  84			pr_err("Missing field from shadow_read_only_field %x\n",
  85			       field + 1);
  86
  87		clear_bit(field, vmx_vmread_bitmap);
  88		if (field & 1)
  89#ifdef CONFIG_X86_64
  90			continue;
  91#else
  92			entry.offset += sizeof(u32);
  93#endif
  94		shadow_read_only_fields[j++] = entry;
  95	}
  96	max_shadow_read_only_fields = j;
  97
  98	for (i = j = 0; i < max_shadow_read_write_fields; i++) {
  99		struct shadow_vmcs_field entry = shadow_read_write_fields[i];
 100		u16 field = entry.encoding;
 101
 102		if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
 103		    (i + 1 == max_shadow_read_write_fields ||
 104		     shadow_read_write_fields[i + 1].encoding != field + 1))
 105			pr_err("Missing field from shadow_read_write_field %x\n",
 106			       field + 1);
 107
 108		WARN_ONCE(field >= GUEST_ES_AR_BYTES &&
 109			  field <= GUEST_TR_AR_BYTES,
 110			  "Update vmcs12_write_any() to drop reserved bits from AR_BYTES");
 111
 112		/*
 113		 * PML and the preemption timer can be emulated, but the
 114		 * processor cannot vmwrite to fields that don't exist
 115		 * on bare metal.
 116		 */
 117		switch (field) {
 118		case GUEST_PML_INDEX:
 119			if (!cpu_has_vmx_pml())
 120				continue;
 121			break;
 122		case VMX_PREEMPTION_TIMER_VALUE:
 123			if (!cpu_has_vmx_preemption_timer())
 124				continue;
 125			break;
 126		case GUEST_INTR_STATUS:
 127			if (!cpu_has_vmx_apicv())
 128				continue;
 129			break;
 130		default:
 131			break;
 132		}
 133
 134		clear_bit(field, vmx_vmwrite_bitmap);
 135		clear_bit(field, vmx_vmread_bitmap);
 136		if (field & 1)
 137#ifdef CONFIG_X86_64
 138			continue;
 139#else
 140			entry.offset += sizeof(u32);
 141#endif
 142		shadow_read_write_fields[j++] = entry;
 143	}
 144	max_shadow_read_write_fields = j;
 145}
 146
 147/*
 148 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
 149 * set the success or error code of an emulated VMX instruction (as specified
 150 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated
 151 * instruction.
 152 */
 153static int nested_vmx_succeed(struct kvm_vcpu *vcpu)
 154{
 155	vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
 156			& ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
 157			    X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
 158	return kvm_skip_emulated_instruction(vcpu);
 159}
 160
 161static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
 162{
 163	vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
 164			& ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
 165			    X86_EFLAGS_SF | X86_EFLAGS_OF))
 166			| X86_EFLAGS_CF);
 167	return kvm_skip_emulated_instruction(vcpu);
 168}
 169
 170static int nested_vmx_failValid(struct kvm_vcpu *vcpu,
 171				u32 vm_instruction_error)
 172{
 173	vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
 174			& ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
 175			    X86_EFLAGS_SF | X86_EFLAGS_OF))
 176			| X86_EFLAGS_ZF);
 177	get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
 178	/*
 179	 * We don't need to force sync to shadow VMCS because
 180	 * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all
 181	 * fields and thus must be synced.
 182	 */
 183	if (nested_vmx_is_evmptr12_set(to_vmx(vcpu)))
 184		to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true;
 185
 186	return kvm_skip_emulated_instruction(vcpu);
 187}
 188
 189static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error)
 190{
 191	struct vcpu_vmx *vmx = to_vmx(vcpu);
 192
 193	/*
 194	 * failValid writes the error number to the current VMCS, which
 195	 * can't be done if there isn't a current VMCS.
 196	 */
 197	if (vmx->nested.current_vmptr == INVALID_GPA &&
 198	    !nested_vmx_is_evmptr12_valid(vmx))
 199		return nested_vmx_failInvalid(vcpu);
 200
 201	return nested_vmx_failValid(vcpu, vm_instruction_error);
 202}
 203
 204static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
 205{
 206	/* TODO: not to reset guest simply here. */
 207	kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
 208	pr_debug_ratelimited("nested vmx abort, indicator %d\n", indicator);
 209}
 210
 211static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
 212{
 213	return fixed_bits_valid(control, low, high);
 214}
 215
 216static inline u64 vmx_control_msr(u32 low, u32 high)
 217{
 218	return low | ((u64)high << 32);
 219}
 220
 221static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
 222{
 223	secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
 224	vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA);
 225	vmx->nested.need_vmcs12_to_shadow_sync = false;
 226}
 227
 228static inline void nested_release_evmcs(struct kvm_vcpu *vcpu)
 229{
 230#ifdef CONFIG_KVM_HYPERV
 231	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
 232	struct vcpu_vmx *vmx = to_vmx(vcpu);
 233
 234	kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map);
 235	vmx->nested.hv_evmcs = NULL;
 
 
 
 236	vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID;
 237
 238	if (hv_vcpu) {
 239		hv_vcpu->nested.pa_page_gpa = INVALID_GPA;
 240		hv_vcpu->nested.vm_id = 0;
 241		hv_vcpu->nested.vp_id = 0;
 242	}
 243#endif
 244}
 245
 246static bool nested_evmcs_handle_vmclear(struct kvm_vcpu *vcpu, gpa_t vmptr)
 247{
 248#ifdef CONFIG_KVM_HYPERV
 249	struct vcpu_vmx *vmx = to_vmx(vcpu);
 250	/*
 251	 * When Enlightened VMEntry is enabled on the calling CPU we treat
 252	 * memory area pointer by vmptr as Enlightened VMCS (as there's no good
 253	 * way to distinguish it from VMCS12) and we must not corrupt it by
 254	 * writing to the non-existent 'launch_state' field. The area doesn't
 255	 * have to be the currently active EVMCS on the calling CPU and there's
 256	 * nothing KVM has to do to transition it from 'active' to 'non-active'
 257	 * state. It is possible that the area will stay mapped as
 258	 * vmx->nested.hv_evmcs but this shouldn't be a problem.
 259	 */
 260	if (!guest_cpuid_has_evmcs(vcpu) ||
 261	    !evmptr_is_valid(nested_get_evmptr(vcpu)))
 262		return false;
 263
 264	if (nested_vmx_evmcs(vmx) && vmptr == vmx->nested.hv_evmcs_vmptr)
 265		nested_release_evmcs(vcpu);
 266
 267	return true;
 268#else
 269	return false;
 270#endif
 271}
 272
 273static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx,
 274				     struct loaded_vmcs *prev)
 275{
 276	struct vmcs_host_state *dest, *src;
 277
 278	if (unlikely(!vmx->guest_state_loaded))
 279		return;
 280
 281	src = &prev->host_state;
 282	dest = &vmx->loaded_vmcs->host_state;
 283
 284	vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base);
 285	dest->ldt_sel = src->ldt_sel;
 286#ifdef CONFIG_X86_64
 287	dest->ds_sel = src->ds_sel;
 288	dest->es_sel = src->es_sel;
 289#endif
 290}
 291
 292static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
 293{
 294	struct vcpu_vmx *vmx = to_vmx(vcpu);
 295	struct loaded_vmcs *prev;
 296	int cpu;
 297
 298	if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs))
 299		return;
 300
 301	cpu = get_cpu();
 302	prev = vmx->loaded_vmcs;
 303	vmx->loaded_vmcs = vmcs;
 304	vmx_vcpu_load_vmcs(vcpu, cpu, prev);
 305	vmx_sync_vmcs_host_state(vmx, prev);
 306	put_cpu();
 307
 308	vcpu->arch.regs_avail = ~VMX_REGS_LAZY_LOAD_SET;
 309
 310	/*
 311	 * All lazily updated registers will be reloaded from VMCS12 on both
 312	 * vmentry and vmexit.
 313	 */
 314	vcpu->arch.regs_dirty = 0;
 315}
 316
 317static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu)
 318{
 319	struct vcpu_vmx *vmx = to_vmx(vcpu);
 320
 321	kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map);
 322	kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map);
 323	kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map);
 324	vmx->nested.pi_desc = NULL;
 325}
 326
 327/*
 328 * Free whatever needs to be freed from vmx->nested when L1 goes down, or
 329 * just stops using VMX.
 330 */
 331static void free_nested(struct kvm_vcpu *vcpu)
 332{
 333	struct vcpu_vmx *vmx = to_vmx(vcpu);
 334
 335	if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01))
 336		vmx_switch_vmcs(vcpu, &vmx->vmcs01);
 337
 338	if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
 339		return;
 340
 341	kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
 342
 343	vmx->nested.vmxon = false;
 344	vmx->nested.smm.vmxon = false;
 345	vmx->nested.vmxon_ptr = INVALID_GPA;
 346	free_vpid(vmx->nested.vpid02);
 347	vmx->nested.posted_intr_nv = -1;
 348	vmx->nested.current_vmptr = INVALID_GPA;
 349	if (enable_shadow_vmcs) {
 350		vmx_disable_shadow_vmcs(vmx);
 351		vmcs_clear(vmx->vmcs01.shadow_vmcs);
 352		free_vmcs(vmx->vmcs01.shadow_vmcs);
 353		vmx->vmcs01.shadow_vmcs = NULL;
 354	}
 355	kfree(vmx->nested.cached_vmcs12);
 356	vmx->nested.cached_vmcs12 = NULL;
 357	kfree(vmx->nested.cached_shadow_vmcs12);
 358	vmx->nested.cached_shadow_vmcs12 = NULL;
 359
 360	nested_put_vmcs12_pages(vcpu);
 
 
 
 
 
 
 
 361
 362	kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
 363
 364	nested_release_evmcs(vcpu);
 365
 366	free_loaded_vmcs(&vmx->nested.vmcs02);
 367}
 368
 369/*
 370 * Ensure that the current vmcs of the logical processor is the
 371 * vmcs01 of the vcpu before calling free_nested().
 372 */
 373void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu)
 374{
 375	vcpu_load(vcpu);
 376	vmx_leave_nested(vcpu);
 377	vcpu_put(vcpu);
 378}
 379
 380#define EPTP_PA_MASK   GENMASK_ULL(51, 12)
 381
 382static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp)
 383{
 384	return VALID_PAGE(root_hpa) &&
 385	       ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK));
 386}
 387
 388static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 389				       gpa_t addr)
 390{
 391	unsigned long roots = 0;
 392	uint i;
 393	struct kvm_mmu_root_info *cached_root;
 394
 395	WARN_ON_ONCE(!mmu_is_nested(vcpu));
 396
 397	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
 398		cached_root = &vcpu->arch.mmu->prev_roots[i];
 399
 400		if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd,
 401					    eptp))
 402			roots |= KVM_MMU_ROOT_PREVIOUS(i);
 403	}
 404	if (roots)
 405		kvm_mmu_invalidate_addr(vcpu, vcpu->arch.mmu, addr, roots);
 406}
 407
 408static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
 409		struct x86_exception *fault)
 410{
 411	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 412	struct vcpu_vmx *vmx = to_vmx(vcpu);
 413	unsigned long exit_qualification;
 414	u32 vm_exit_reason;
 
 415
 416	if (vmx->nested.pml_full) {
 417		vm_exit_reason = EXIT_REASON_PML_FULL;
 418		vmx->nested.pml_full = false;
 419
 420		/*
 421		 * It should be impossible to trigger a nested PML Full VM-Exit
 422		 * for anything other than an EPT Violation from L2.  KVM *can*
 423		 * trigger nEPT page fault injection in response to an EPT
 424		 * Misconfig, e.g. if the MMIO SPTE was stale and L1's EPT
 425		 * tables also changed, but KVM should not treat EPT Misconfig
 426		 * VM-Exits as writes.
 427		 */
 428		WARN_ON_ONCE(vmx->exit_reason.basic != EXIT_REASON_EPT_VIOLATION);
 429
 430		/*
 431		 * PML Full and EPT Violation VM-Exits both use bit 12 to report
 432		 * "NMI unblocking due to IRET", i.e. the bit can be propagated
 433		 * as-is from the original EXIT_QUALIFICATION.
 434		 */
 435		exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI;
 436	} else {
 437		if (fault->error_code & PFERR_RSVD_MASK) {
 438			vm_exit_reason = EXIT_REASON_EPT_MISCONFIG;
 439			exit_qualification = 0;
 440		} else {
 441			exit_qualification = fault->exit_qualification;
 442			exit_qualification |= vmx_get_exit_qual(vcpu) &
 443					      (EPT_VIOLATION_GVA_IS_VALID |
 444					       EPT_VIOLATION_GVA_TRANSLATED);
 445			vm_exit_reason = EXIT_REASON_EPT_VIOLATION;
 446		}
 447
 448		/*
 449		 * Although the caller (kvm_inject_emulated_page_fault) would
 450		 * have already synced the faulting address in the shadow EPT
 451		 * tables for the current EPTP12, we also need to sync it for
 452		 * any other cached EPTP02s based on the same EP4TA, since the
 453		 * TLB associates mappings to the EP4TA rather than the full EPTP.
 454		 */
 455		nested_ept_invalidate_addr(vcpu, vmcs12->ept_pointer,
 456					   fault->address);
 457	}
 458
 459	nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification);
 460	vmcs12->guest_physical_address = fault->address;
 461}
 462
 463static void nested_ept_new_eptp(struct kvm_vcpu *vcpu)
 464{
 465	struct vcpu_vmx *vmx = to_vmx(vcpu);
 466	bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT;
 467	int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps);
 468
 469	kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level,
 470				nested_ept_ad_enabled(vcpu),
 471				nested_ept_get_eptp(vcpu));
 472}
 473
 474static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
 475{
 476	WARN_ON(mmu_is_nested(vcpu));
 477
 478	vcpu->arch.mmu = &vcpu->arch.guest_mmu;
 479	nested_ept_new_eptp(vcpu);
 480	vcpu->arch.mmu->get_guest_pgd     = nested_ept_get_eptp;
 481	vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
 482	vcpu->arch.mmu->get_pdptr         = kvm_pdptr_read;
 483
 484	vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
 485}
 486
 487static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
 488{
 489	vcpu->arch.mmu = &vcpu->arch.root_mmu;
 490	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
 491}
 492
 493static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
 494					    u16 error_code)
 495{
 496	bool inequality, bit;
 497
 498	bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
 499	inequality =
 500		(error_code & vmcs12->page_fault_error_code_mask) !=
 501		 vmcs12->page_fault_error_code_match;
 502	return inequality ^ bit;
 503}
 504
 505static bool nested_vmx_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector,
 506					   u32 error_code)
 507{
 508	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 509
 510	/*
 511	 * Drop bits 31:16 of the error code when performing the #PF mask+match
 512	 * check.  All VMCS fields involved are 32 bits, but Intel CPUs never
 513	 * set bits 31:16 and VMX disallows setting bits 31:16 in the injected
 514	 * error code.  Including the to-be-dropped bits in the check might
 515	 * result in an "impossible" or missed exit from L1's perspective.
 516	 */
 517	if (vector == PF_VECTOR)
 518		return nested_vmx_is_page_fault_vmexit(vmcs12, (u16)error_code);
 519
 520	return (vmcs12->exception_bitmap & (1u << vector));
 521}
 522
 523static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu,
 524					       struct vmcs12 *vmcs12)
 525{
 526	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
 527		return 0;
 528
 529	if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) ||
 530	    CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b)))
 531		return -EINVAL;
 532
 533	return 0;
 534}
 535
 536static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
 537						struct vmcs12 *vmcs12)
 538{
 539	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
 540		return 0;
 541
 542	if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap)))
 543		return -EINVAL;
 544
 545	return 0;
 546}
 547
 548static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
 549						struct vmcs12 *vmcs12)
 550{
 551	if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
 552		return 0;
 553
 554	if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)))
 555		return -EINVAL;
 556
 557	return 0;
 558}
 559
 560/*
 561 * For x2APIC MSRs, ignore the vmcs01 bitmap.  L1 can enable x2APIC without L1
 562 * itself utilizing x2APIC.  All MSRs were previously set to be intercepted,
 563 * only the "disable intercept" case needs to be handled.
 564 */
 565static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1,
 566							unsigned long *msr_bitmap_l0,
 567							u32 msr, int type)
 568{
 569	if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr))
 570		vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr);
 571
 572	if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr))
 573		vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr);
 574}
 575
 576static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap)
 577{
 578	int msr;
 579
 580	for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
 581		unsigned word = msr / BITS_PER_LONG;
 582
 583		msr_bitmap[word] = ~0;
 584		msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
 585	}
 586}
 587
 588#define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw)					\
 589static inline									\
 590void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx,			\
 591					 unsigned long *msr_bitmap_l1,		\
 592					 unsigned long *msr_bitmap_l0, u32 msr)	\
 593{										\
 594	if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) ||		\
 595	    vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr))			\
 596		vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr);			\
 597	else									\
 598		vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr);			\
 599}
 600BUILD_NVMX_MSR_INTERCEPT_HELPER(read)
 601BUILD_NVMX_MSR_INTERCEPT_HELPER(write)
 602
 603static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx,
 604						    unsigned long *msr_bitmap_l1,
 605						    unsigned long *msr_bitmap_l0,
 606						    u32 msr, int types)
 607{
 608	if (types & MSR_TYPE_R)
 609		nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1,
 610						  msr_bitmap_l0, msr);
 611	if (types & MSR_TYPE_W)
 612		nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1,
 613						   msr_bitmap_l0, msr);
 614}
 615
 616/*
 617 * Merge L0's and L1's MSR bitmap, return false to indicate that
 618 * we do not use the hardware.
 619 */
 620static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
 621						 struct vmcs12 *vmcs12)
 622{
 623	struct vcpu_vmx *vmx = to_vmx(vcpu);
 624	int msr;
 625	unsigned long *msr_bitmap_l1;
 626	unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap;
 627	struct kvm_host_map map;
 628
 629	/* Nothing to do if the MSR bitmap is not in use.  */
 630	if (!cpu_has_vmx_msr_bitmap() ||
 631	    !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
 632		return false;
 633
 634	/*
 635	 * MSR bitmap update can be skipped when:
 636	 * - MSR bitmap for L1 hasn't changed.
 637	 * - Nested hypervisor (L1) is attempting to launch the same L2 as
 638	 *   before.
 639	 * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature
 640	 *   and tells KVM (L0) there were no changes in MSR bitmap for L2.
 641	 */
 642	if (!vmx->nested.force_msr_bitmap_recalc) {
 643		struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx);
 644
 645		if (evmcs && evmcs->hv_enlightenments_control.msr_bitmap &&
 646		    evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP)
 647			return true;
 648	}
 649
 650	if (kvm_vcpu_map_readonly(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), &map))
 651		return false;
 652
 653	msr_bitmap_l1 = (unsigned long *)map.hva;
 654
 655	/*
 656	 * To keep the control flow simple, pay eight 8-byte writes (sixteen
 657	 * 4-byte writes on 32-bit systems) up front to enable intercepts for
 658	 * the x2APIC MSR range and selectively toggle those relevant to L2.
 659	 */
 660	enable_x2apic_msr_intercepts(msr_bitmap_l0);
 661
 662	if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
 663		if (nested_cpu_has_apic_reg_virt(vmcs12)) {
 664			/*
 665			 * L0 need not intercept reads for MSRs between 0x800
 666			 * and 0x8ff, it just lets the processor take the value
 667			 * from the virtual-APIC page; take those 256 bits
 668			 * directly from the L1 bitmap.
 669			 */
 670			for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
 671				unsigned word = msr / BITS_PER_LONG;
 672
 673				msr_bitmap_l0[word] = msr_bitmap_l1[word];
 674			}
 675		}
 676
 677		nested_vmx_disable_intercept_for_x2apic_msr(
 678			msr_bitmap_l1, msr_bitmap_l0,
 679			X2APIC_MSR(APIC_TASKPRI),
 680			MSR_TYPE_R | MSR_TYPE_W);
 681
 682		if (nested_cpu_has_vid(vmcs12)) {
 683			nested_vmx_disable_intercept_for_x2apic_msr(
 684				msr_bitmap_l1, msr_bitmap_l0,
 685				X2APIC_MSR(APIC_EOI),
 686				MSR_TYPE_W);
 687			nested_vmx_disable_intercept_for_x2apic_msr(
 688				msr_bitmap_l1, msr_bitmap_l0,
 689				X2APIC_MSR(APIC_SELF_IPI),
 690				MSR_TYPE_W);
 691		}
 692	}
 693
 694	/*
 695	 * Always check vmcs01's bitmap to honor userspace MSR filters and any
 696	 * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through.
 697	 */
 698#ifdef CONFIG_X86_64
 699	nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
 700					 MSR_FS_BASE, MSR_TYPE_RW);
 701
 702	nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
 703					 MSR_GS_BASE, MSR_TYPE_RW);
 704
 705	nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
 706					 MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
 707#endif
 708	nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
 709					 MSR_IA32_SPEC_CTRL, MSR_TYPE_RW);
 710
 711	nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
 712					 MSR_IA32_PRED_CMD, MSR_TYPE_W);
 713
 714	nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
 715					 MSR_IA32_FLUSH_CMD, MSR_TYPE_W);
 716
 717	kvm_vcpu_unmap(vcpu, &map);
 718
 719	vmx->nested.force_msr_bitmap_recalc = false;
 720
 721	return true;
 722}
 723
 724static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu,
 725				       struct vmcs12 *vmcs12)
 726{
 727	struct vcpu_vmx *vmx = to_vmx(vcpu);
 728	struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache;
 729
 730	if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
 731	    vmcs12->vmcs_link_pointer == INVALID_GPA)
 732		return;
 733
 734	if (ghc->gpa != vmcs12->vmcs_link_pointer &&
 735	    kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc,
 736				      vmcs12->vmcs_link_pointer, VMCS12_SIZE))
 737		return;
 738
 739	kvm_read_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu),
 740			      VMCS12_SIZE);
 741}
 742
 743static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu,
 744					      struct vmcs12 *vmcs12)
 745{
 746	struct vcpu_vmx *vmx = to_vmx(vcpu);
 747	struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache;
 748
 749	if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
 750	    vmcs12->vmcs_link_pointer == INVALID_GPA)
 751		return;
 752
 753	if (ghc->gpa != vmcs12->vmcs_link_pointer &&
 754	    kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc,
 755				      vmcs12->vmcs_link_pointer, VMCS12_SIZE))
 756		return;
 757
 758	kvm_write_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu),
 759			       VMCS12_SIZE);
 760}
 761
 762/*
 763 * In nested virtualization, check if L1 has set
 764 * VM_EXIT_ACK_INTR_ON_EXIT
 765 */
 766static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
 767{
 768	return get_vmcs12(vcpu)->vm_exit_controls &
 769		VM_EXIT_ACK_INTR_ON_EXIT;
 770}
 771
 772static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu,
 773					  struct vmcs12 *vmcs12)
 774{
 775	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
 776	    CC(!page_address_valid(vcpu, vmcs12->apic_access_addr)))
 777		return -EINVAL;
 778	else
 779		return 0;
 780}
 781
 782static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
 783					   struct vmcs12 *vmcs12)
 784{
 785	if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
 786	    !nested_cpu_has_apic_reg_virt(vmcs12) &&
 787	    !nested_cpu_has_vid(vmcs12) &&
 788	    !nested_cpu_has_posted_intr(vmcs12))
 789		return 0;
 790
 791	/*
 792	 * If virtualize x2apic mode is enabled,
 793	 * virtualize apic access must be disabled.
 794	 */
 795	if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) &&
 796	       nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)))
 797		return -EINVAL;
 798
 799	/*
 800	 * If virtual interrupt delivery is enabled,
 801	 * we must exit on external interrupts.
 802	 */
 803	if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu)))
 804		return -EINVAL;
 805
 806	/*
 807	 * bits 15:8 should be zero in posted_intr_nv,
 808	 * the descriptor address has been already checked
 809	 * in nested_get_vmcs12_pages.
 810	 *
 811	 * bits 5:0 of posted_intr_desc_addr should be zero.
 812	 */
 813	if (nested_cpu_has_posted_intr(vmcs12) &&
 814	   (CC(!nested_cpu_has_vid(vmcs12)) ||
 815	    CC(!nested_exit_intr_ack_set(vcpu)) ||
 816	    CC((vmcs12->posted_intr_nv & 0xff00)) ||
 817	    CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64))))
 818		return -EINVAL;
 819
 820	/* tpr shadow is needed by all apicv features. */
 821	if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)))
 822		return -EINVAL;
 823
 824	return 0;
 825}
 826
 827static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
 828				       u32 count, u64 addr)
 829{
 830	if (count == 0)
 831		return 0;
 832
 833	if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) ||
 834	    !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1)))
 835		return -EINVAL;
 836
 837	return 0;
 838}
 839
 840static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu,
 841						     struct vmcs12 *vmcs12)
 842{
 843	if (CC(nested_vmx_check_msr_switch(vcpu,
 844					   vmcs12->vm_exit_msr_load_count,
 845					   vmcs12->vm_exit_msr_load_addr)) ||
 846	    CC(nested_vmx_check_msr_switch(vcpu,
 847					   vmcs12->vm_exit_msr_store_count,
 848					   vmcs12->vm_exit_msr_store_addr)))
 849		return -EINVAL;
 850
 851	return 0;
 852}
 853
 854static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu,
 855                                                      struct vmcs12 *vmcs12)
 856{
 857	if (CC(nested_vmx_check_msr_switch(vcpu,
 858					   vmcs12->vm_entry_msr_load_count,
 859					   vmcs12->vm_entry_msr_load_addr)))
 860                return -EINVAL;
 861
 862	return 0;
 863}
 864
 865static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu,
 866					 struct vmcs12 *vmcs12)
 867{
 868	if (!nested_cpu_has_pml(vmcs12))
 869		return 0;
 870
 871	if (CC(!nested_cpu_has_ept(vmcs12)) ||
 872	    CC(!page_address_valid(vcpu, vmcs12->pml_address)))
 873		return -EINVAL;
 874
 875	return 0;
 876}
 877
 878static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu,
 879							struct vmcs12 *vmcs12)
 880{
 881	if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&
 882	       !nested_cpu_has_ept(vmcs12)))
 883		return -EINVAL;
 884	return 0;
 885}
 886
 887static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu,
 888							 struct vmcs12 *vmcs12)
 889{
 890	if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&
 891	       !nested_cpu_has_ept(vmcs12)))
 892		return -EINVAL;
 893	return 0;
 894}
 895
 896static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu,
 897						 struct vmcs12 *vmcs12)
 898{
 899	if (!nested_cpu_has_shadow_vmcs(vmcs12))
 900		return 0;
 901
 902	if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) ||
 903	    CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap)))
 904		return -EINVAL;
 905
 906	return 0;
 907}
 908
 909static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
 910				       struct vmx_msr_entry *e)
 911{
 912	/* x2APIC MSR accesses are not allowed */
 913	if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8))
 914		return -EINVAL;
 915	if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */
 916	    CC(e->index == MSR_IA32_UCODE_REV))
 917		return -EINVAL;
 918	if (CC(e->reserved != 0))
 919		return -EINVAL;
 920	return 0;
 921}
 922
 923static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
 924				     struct vmx_msr_entry *e)
 925{
 926	if (CC(e->index == MSR_FS_BASE) ||
 927	    CC(e->index == MSR_GS_BASE) ||
 928	    CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */
 929	    nested_vmx_msr_check_common(vcpu, e))
 930		return -EINVAL;
 931	return 0;
 932}
 933
 934static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
 935				      struct vmx_msr_entry *e)
 936{
 937	if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */
 938	    nested_vmx_msr_check_common(vcpu, e))
 939		return -EINVAL;
 940	return 0;
 941}
 942
 943static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu)
 944{
 945	struct vcpu_vmx *vmx = to_vmx(vcpu);
 946	u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
 947				       vmx->nested.msrs.misc_high);
 948
 949	return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER;
 950}
 951
 952/*
 953 * Load guest's/host's msr at nested entry/exit.
 954 * return 0 for success, entry index for failure.
 955 *
 956 * One of the failure modes for MSR load/store is when a list exceeds the
 957 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch
 958 * as possible, process all valid entries before failing rather than precheck
 959 * for a capacity violation.
 960 */
 961static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
 962{
 963	u32 i;
 964	struct vmx_msr_entry e;
 965	u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu);
 966
 967	for (i = 0; i < count; i++) {
 968		if (unlikely(i >= max_msr_list_size))
 969			goto fail;
 970
 971		if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
 972					&e, sizeof(e))) {
 973			pr_debug_ratelimited(
 974				"%s cannot read MSR entry (%u, 0x%08llx)\n",
 975				__func__, i, gpa + i * sizeof(e));
 976			goto fail;
 977		}
 978		if (nested_vmx_load_msr_check(vcpu, &e)) {
 979			pr_debug_ratelimited(
 980				"%s check failed (%u, 0x%x, 0x%x)\n",
 981				__func__, i, e.index, e.reserved);
 982			goto fail;
 983		}
 984		if (kvm_set_msr_with_filter(vcpu, e.index, e.value)) {
 985			pr_debug_ratelimited(
 986				"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
 987				__func__, i, e.index, e.value);
 988			goto fail;
 989		}
 990	}
 991	return 0;
 992fail:
 993	/* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */
 994	return i + 1;
 995}
 996
 997static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu,
 998					    u32 msr_index,
 999					    u64 *data)
1000{
1001	struct vcpu_vmx *vmx = to_vmx(vcpu);
1002
1003	/*
1004	 * If the L0 hypervisor stored a more accurate value for the TSC that
1005	 * does not include the time taken for emulation of the L2->L1
1006	 * VM-exit in L0, use the more accurate value.
1007	 */
1008	if (msr_index == MSR_IA32_TSC) {
1009		int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest,
1010						    MSR_IA32_TSC);
1011
1012		if (i >= 0) {
1013			u64 val = vmx->msr_autostore.guest.val[i].value;
1014
1015			*data = kvm_read_l1_tsc(vcpu, val);
1016			return true;
1017		}
1018	}
1019
1020	if (kvm_get_msr_with_filter(vcpu, msr_index, data)) {
1021		pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__,
1022			msr_index);
1023		return false;
1024	}
1025	return true;
1026}
1027
1028static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i,
1029				     struct vmx_msr_entry *e)
1030{
1031	if (kvm_vcpu_read_guest(vcpu,
1032				gpa + i * sizeof(*e),
1033				e, 2 * sizeof(u32))) {
1034		pr_debug_ratelimited(
1035			"%s cannot read MSR entry (%u, 0x%08llx)\n",
1036			__func__, i, gpa + i * sizeof(*e));
1037		return false;
1038	}
1039	if (nested_vmx_store_msr_check(vcpu, e)) {
1040		pr_debug_ratelimited(
1041			"%s check failed (%u, 0x%x, 0x%x)\n",
1042			__func__, i, e->index, e->reserved);
1043		return false;
1044	}
1045	return true;
1046}
1047
1048static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
1049{
1050	u64 data;
1051	u32 i;
1052	struct vmx_msr_entry e;
1053	u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu);
1054
1055	for (i = 0; i < count; i++) {
1056		if (unlikely(i >= max_msr_list_size))
1057			return -EINVAL;
1058
1059		if (!read_and_check_msr_entry(vcpu, gpa, i, &e))
1060			return -EINVAL;
1061
1062		if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data))
1063			return -EINVAL;
1064
1065		if (kvm_vcpu_write_guest(vcpu,
1066					 gpa + i * sizeof(e) +
1067					     offsetof(struct vmx_msr_entry, value),
1068					 &data, sizeof(data))) {
1069			pr_debug_ratelimited(
1070				"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
1071				__func__, i, e.index, data);
1072			return -EINVAL;
1073		}
1074	}
1075	return 0;
1076}
1077
1078static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index)
1079{
1080	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1081	u32 count = vmcs12->vm_exit_msr_store_count;
1082	u64 gpa = vmcs12->vm_exit_msr_store_addr;
1083	struct vmx_msr_entry e;
1084	u32 i;
1085
1086	for (i = 0; i < count; i++) {
1087		if (!read_and_check_msr_entry(vcpu, gpa, i, &e))
1088			return false;
1089
1090		if (e.index == msr_index)
1091			return true;
1092	}
1093	return false;
1094}
1095
1096static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu,
1097					   u32 msr_index)
1098{
1099	struct vcpu_vmx *vmx = to_vmx(vcpu);
1100	struct vmx_msrs *autostore = &vmx->msr_autostore.guest;
1101	bool in_vmcs12_store_list;
1102	int msr_autostore_slot;
1103	bool in_autostore_list;
1104	int last;
1105
1106	msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index);
1107	in_autostore_list = msr_autostore_slot >= 0;
1108	in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index);
1109
1110	if (in_vmcs12_store_list && !in_autostore_list) {
1111		if (autostore->nr == MAX_NR_LOADSTORE_MSRS) {
1112			/*
1113			 * Emulated VMEntry does not fail here.  Instead a less
1114			 * accurate value will be returned by
1115			 * nested_vmx_get_vmexit_msr_value() by reading KVM's
1116			 * internal MSR state instead of reading the value from
1117			 * the vmcs02 VMExit MSR-store area.
1118			 */
1119			pr_warn_ratelimited(
1120				"Not enough msr entries in msr_autostore.  Can't add msr %x\n",
1121				msr_index);
1122			return;
1123		}
1124		last = autostore->nr++;
1125		autostore->val[last].index = msr_index;
1126	} else if (!in_vmcs12_store_list && in_autostore_list) {
1127		last = --autostore->nr;
1128		autostore->val[msr_autostore_slot] = autostore->val[last];
1129	}
1130}
1131
1132/*
1133 * Load guest's/host's cr3 at nested entry/exit.  @nested_ept is true if we are
1134 * emulating VM-Entry into a guest with EPT enabled.  On failure, the expected
1135 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to
1136 * @entry_failure_code.
1137 */
1138static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
1139			       bool nested_ept, bool reload_pdptrs,
1140			       enum vm_entry_failure_code *entry_failure_code)
1141{
1142	if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) {
1143		*entry_failure_code = ENTRY_FAIL_DEFAULT;
1144		return -EINVAL;
1145	}
1146
1147	/*
1148	 * If PAE paging and EPT are both on, CR3 is not used by the CPU and
1149	 * must not be dereferenced.
1150	 */
1151	if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) &&
1152	    CC(!load_pdptrs(vcpu, cr3))) {
1153		*entry_failure_code = ENTRY_FAIL_PDPTE;
1154		return -EINVAL;
1155	}
1156
1157	vcpu->arch.cr3 = cr3;
1158	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
1159
1160	/* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */
1161	kvm_init_mmu(vcpu);
1162
1163	if (!nested_ept)
1164		kvm_mmu_new_pgd(vcpu, cr3);
1165
1166	return 0;
1167}
1168
1169/*
1170 * Returns if KVM is able to config CPU to tag TLB entries
1171 * populated by L2 differently than TLB entries populated
1172 * by L1.
1173 *
1174 * If L0 uses EPT, L1 and L2 run with different EPTP because
1175 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries
1176 * are tagged with different EPTP.
1177 *
1178 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged
1179 * with different VPID (L1 entries are tagged with vmx->vpid
1180 * while L2 entries are tagged with vmx->nested.vpid02).
1181 */
1182static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu)
1183{
1184	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1185
1186	return enable_ept ||
1187	       (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02);
1188}
1189
1190static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu,
1191					    struct vmcs12 *vmcs12,
1192					    bool is_vmenter)
1193{
1194	struct vcpu_vmx *vmx = to_vmx(vcpu);
1195
1196	/* Handle pending Hyper-V TLB flush requests */
1197	kvm_hv_nested_transtion_tlb_flush(vcpu, enable_ept);
1198
1199	/*
1200	 * If VPID is disabled, then guest TLB accesses use VPID=0, i.e. the
1201	 * same VPID as the host, and so architecturally, linear and combined
1202	 * mappings for VPID=0 must be flushed at VM-Enter and VM-Exit.  KVM
1203	 * emulates L2 sharing L1's VPID=0 by using vpid01 while running L2,
1204	 * and so KVM must also emulate TLB flush of VPID=0, i.e. vpid01.  This
1205	 * is required if VPID is disabled in KVM, as a TLB flush (there are no
1206	 * VPIDs) still occurs from L1's perspective, and KVM may need to
1207	 * synchronize the MMU in response to the guest TLB flush.
1208	 *
1209	 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use.
1210	 * EPT is a special snowflake, as guest-physical mappings aren't
1211	 * flushed on VPID invalidations, including VM-Enter or VM-Exit with
1212	 * VPID disabled.  As a result, KVM _never_ needs to sync nEPT
1213	 * entries on VM-Enter because L1 can't rely on VM-Enter to flush
1214	 * those mappings.
1215	 */
1216	if (!nested_cpu_has_vpid(vmcs12)) {
1217		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1218		return;
1219	}
1220
1221	/* L2 should never have a VPID if VPID is disabled. */
1222	WARN_ON(!enable_vpid);
1223
1224	/*
1225	 * VPID is enabled and in use by vmcs12.  If vpid12 is changing, then
1226	 * emulate a guest TLB flush as KVM does not track vpid12 history nor
1227	 * is the VPID incorporated into the MMU context.  I.e. KVM must assume
1228	 * that the new vpid12 has never been used and thus represents a new
1229	 * guest ASID that cannot have entries in the TLB.
1230	 */
1231	if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
1232		vmx->nested.last_vpid = vmcs12->virtual_processor_id;
1233		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1234		return;
1235	}
1236
1237	/*
1238	 * If VPID is enabled, used by vmc12, and vpid12 is not changing but
1239	 * does not have a unique TLB tag (ASID), i.e. EPT is disabled and
1240	 * KVM was unable to allocate a VPID for L2, flush the current context
1241	 * as the effective ASID is common to both L1 and L2.
1242	 */
1243	if (!nested_has_guest_tlb_tag(vcpu))
1244		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1245}
1246
1247static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
1248{
1249	superset &= mask;
1250	subset &= mask;
1251
1252	return (superset | subset) == superset;
1253}
1254
1255static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
1256{
1257	const u64 feature_bits = VMX_BASIC_DUAL_MONITOR_TREATMENT |
1258				 VMX_BASIC_INOUT |
1259				 VMX_BASIC_TRUE_CTLS;
1260
1261	const u64 reserved_bits = GENMASK_ULL(63, 56) |
1262				  GENMASK_ULL(47, 45) |
1263				  BIT_ULL(31);
1264
1265	u64 vmx_basic = vmcs_config.nested.basic;
1266
1267	BUILD_BUG_ON(feature_bits & reserved_bits);
1268
1269	/*
1270	 * Except for 32BIT_PHYS_ADDR_ONLY, which is an anti-feature bit (has
1271	 * inverted polarity), the incoming value must not set feature bits or
1272	 * reserved bits that aren't allowed/supported by KVM.  Fields, i.e.
1273	 * multi-bit values, are explicitly checked below.
1274	 */
1275	if (!is_bitwise_subset(vmx_basic, data, feature_bits | reserved_bits))
1276		return -EINVAL;
1277
1278	/*
1279	 * KVM does not emulate a version of VMX that constrains physical
1280	 * addresses of VMX structures (e.g. VMCS) to 32-bits.
1281	 */
1282	if (data & VMX_BASIC_32BIT_PHYS_ADDR_ONLY)
1283		return -EINVAL;
1284
1285	if (vmx_basic_vmcs_revision_id(vmx_basic) !=
1286	    vmx_basic_vmcs_revision_id(data))
1287		return -EINVAL;
1288
1289	if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
1290		return -EINVAL;
1291
1292	vmx->nested.msrs.basic = data;
1293	return 0;
1294}
1295
1296static void vmx_get_control_msr(struct nested_vmx_msrs *msrs, u32 msr_index,
1297				u32 **low, u32 **high)
1298{
1299	switch (msr_index) {
1300	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1301		*low = &msrs->pinbased_ctls_low;
1302		*high = &msrs->pinbased_ctls_high;
1303		break;
1304	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1305		*low = &msrs->procbased_ctls_low;
1306		*high = &msrs->procbased_ctls_high;
1307		break;
1308	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1309		*low = &msrs->exit_ctls_low;
1310		*high = &msrs->exit_ctls_high;
1311		break;
1312	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1313		*low = &msrs->entry_ctls_low;
1314		*high = &msrs->entry_ctls_high;
1315		break;
1316	case MSR_IA32_VMX_PROCBASED_CTLS2:
1317		*low = &msrs->secondary_ctls_low;
1318		*high = &msrs->secondary_ctls_high;
1319		break;
1320	default:
1321		BUG();
1322	}
1323}
1324
1325static int
1326vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1327{
1328	u32 *lowp, *highp;
1329	u64 supported;
1330
1331	vmx_get_control_msr(&vmcs_config.nested, msr_index, &lowp, &highp);
1332
1333	supported = vmx_control_msr(*lowp, *highp);
1334
1335	/* Check must-be-1 bits are still 1. */
1336	if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
1337		return -EINVAL;
1338
1339	/* Check must-be-0 bits are still 0. */
1340	if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
1341		return -EINVAL;
1342
1343	vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp);
1344	*lowp = data;
1345	*highp = data >> 32;
1346	return 0;
1347}
1348
1349static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
1350{
1351	const u64 feature_bits = VMX_MISC_SAVE_EFER_LMA |
1352				 VMX_MISC_ACTIVITY_HLT |
1353				 VMX_MISC_ACTIVITY_SHUTDOWN |
1354				 VMX_MISC_ACTIVITY_WAIT_SIPI |
1355				 VMX_MISC_INTEL_PT |
1356				 VMX_MISC_RDMSR_IN_SMM |
1357				 VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
1358				 VMX_MISC_VMXOFF_BLOCK_SMI |
1359				 VMX_MISC_ZERO_LEN_INS;
1360
1361	const u64 reserved_bits = BIT_ULL(31) | GENMASK_ULL(13, 9);
1362
1363	u64 vmx_misc = vmx_control_msr(vmcs_config.nested.misc_low,
1364				       vmcs_config.nested.misc_high);
1365
1366	BUILD_BUG_ON(feature_bits & reserved_bits);
1367
1368	/*
1369	 * The incoming value must not set feature bits or reserved bits that
1370	 * aren't allowed/supported by KVM.  Fields, i.e. multi-bit values, are
1371	 * explicitly checked below.
1372	 */
1373	if (!is_bitwise_subset(vmx_misc, data, feature_bits | reserved_bits))
1374		return -EINVAL;
1375
1376	if ((vmx->nested.msrs.pinbased_ctls_high &
1377	     PIN_BASED_VMX_PREEMPTION_TIMER) &&
1378	    vmx_misc_preemption_timer_rate(data) !=
1379	    vmx_misc_preemption_timer_rate(vmx_misc))
1380		return -EINVAL;
1381
1382	if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
1383		return -EINVAL;
1384
1385	if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
1386		return -EINVAL;
1387
1388	if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
1389		return -EINVAL;
1390
1391	vmx->nested.msrs.misc_low = data;
1392	vmx->nested.msrs.misc_high = data >> 32;
1393
1394	return 0;
1395}
1396
1397static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
1398{
1399	u64 vmx_ept_vpid_cap = vmx_control_msr(vmcs_config.nested.ept_caps,
1400					       vmcs_config.nested.vpid_caps);
1401
1402	/* Every bit is either reserved or a feature bit. */
1403	if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
1404		return -EINVAL;
1405
1406	vmx->nested.msrs.ept_caps = data;
1407	vmx->nested.msrs.vpid_caps = data >> 32;
1408	return 0;
1409}
1410
1411static u64 *vmx_get_fixed0_msr(struct nested_vmx_msrs *msrs, u32 msr_index)
1412{
1413	switch (msr_index) {
1414	case MSR_IA32_VMX_CR0_FIXED0:
1415		return &msrs->cr0_fixed0;
1416	case MSR_IA32_VMX_CR4_FIXED0:
1417		return &msrs->cr4_fixed0;
1418	default:
1419		BUG();
1420	}
1421}
1422
1423static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1424{
1425	const u64 *msr = vmx_get_fixed0_msr(&vmcs_config.nested, msr_index);
1426
1427	/*
1428	 * 1 bits (which indicates bits which "must-be-1" during VMX operation)
1429	 * must be 1 in the restored value.
1430	 */
1431	if (!is_bitwise_subset(data, *msr, -1ULL))
1432		return -EINVAL;
1433
1434	*vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data;
1435	return 0;
1436}
1437
1438/*
1439 * Called when userspace is restoring VMX MSRs.
1440 *
1441 * Returns 0 on success, non-0 otherwise.
1442 */
1443int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1444{
1445	struct vcpu_vmx *vmx = to_vmx(vcpu);
1446
1447	/*
1448	 * Don't allow changes to the VMX capability MSRs while the vCPU
1449	 * is in VMX operation.
1450	 */
1451	if (vmx->nested.vmxon)
1452		return -EBUSY;
1453
1454	switch (msr_index) {
1455	case MSR_IA32_VMX_BASIC:
1456		return vmx_restore_vmx_basic(vmx, data);
1457	case MSR_IA32_VMX_PINBASED_CTLS:
1458	case MSR_IA32_VMX_PROCBASED_CTLS:
1459	case MSR_IA32_VMX_EXIT_CTLS:
1460	case MSR_IA32_VMX_ENTRY_CTLS:
1461		/*
1462		 * The "non-true" VMX capability MSRs are generated from the
1463		 * "true" MSRs, so we do not support restoring them directly.
1464		 *
1465		 * If userspace wants to emulate VMX_BASIC[55]=0, userspace
1466		 * should restore the "true" MSRs with the must-be-1 bits
1467		 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
1468		 * DEFAULT SETTINGS".
1469		 */
1470		return -EINVAL;
1471	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1472	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1473	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1474	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1475	case MSR_IA32_VMX_PROCBASED_CTLS2:
1476		return vmx_restore_control_msr(vmx, msr_index, data);
1477	case MSR_IA32_VMX_MISC:
1478		return vmx_restore_vmx_misc(vmx, data);
1479	case MSR_IA32_VMX_CR0_FIXED0:
1480	case MSR_IA32_VMX_CR4_FIXED0:
1481		return vmx_restore_fixed0_msr(vmx, msr_index, data);
1482	case MSR_IA32_VMX_CR0_FIXED1:
1483	case MSR_IA32_VMX_CR4_FIXED1:
1484		/*
1485		 * These MSRs are generated based on the vCPU's CPUID, so we
1486		 * do not support restoring them directly.
1487		 */
1488		return -EINVAL;
1489	case MSR_IA32_VMX_EPT_VPID_CAP:
1490		return vmx_restore_vmx_ept_vpid_cap(vmx, data);
1491	case MSR_IA32_VMX_VMCS_ENUM:
1492		vmx->nested.msrs.vmcs_enum = data;
1493		return 0;
1494	case MSR_IA32_VMX_VMFUNC:
1495		if (data & ~vmcs_config.nested.vmfunc_controls)
1496			return -EINVAL;
1497		vmx->nested.msrs.vmfunc_controls = data;
1498		return 0;
1499	default:
1500		/*
1501		 * The rest of the VMX capability MSRs do not support restore.
1502		 */
1503		return -EINVAL;
1504	}
1505}
1506
1507/* Returns 0 on success, non-0 otherwise. */
1508int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata)
1509{
1510	switch (msr_index) {
1511	case MSR_IA32_VMX_BASIC:
1512		*pdata = msrs->basic;
1513		break;
1514	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1515	case MSR_IA32_VMX_PINBASED_CTLS:
1516		*pdata = vmx_control_msr(
1517			msrs->pinbased_ctls_low,
1518			msrs->pinbased_ctls_high);
1519		if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
1520			*pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
1521		break;
1522	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1523	case MSR_IA32_VMX_PROCBASED_CTLS:
1524		*pdata = vmx_control_msr(
1525			msrs->procbased_ctls_low,
1526			msrs->procbased_ctls_high);
1527		if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
1528			*pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
1529		break;
1530	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1531	case MSR_IA32_VMX_EXIT_CTLS:
1532		*pdata = vmx_control_msr(
1533			msrs->exit_ctls_low,
1534			msrs->exit_ctls_high);
1535		if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
1536			*pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
1537		break;
1538	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1539	case MSR_IA32_VMX_ENTRY_CTLS:
1540		*pdata = vmx_control_msr(
1541			msrs->entry_ctls_low,
1542			msrs->entry_ctls_high);
1543		if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
1544			*pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
1545		break;
1546	case MSR_IA32_VMX_MISC:
1547		*pdata = vmx_control_msr(
1548			msrs->misc_low,
1549			msrs->misc_high);
1550		break;
1551	case MSR_IA32_VMX_CR0_FIXED0:
1552		*pdata = msrs->cr0_fixed0;
1553		break;
1554	case MSR_IA32_VMX_CR0_FIXED1:
1555		*pdata = msrs->cr0_fixed1;
1556		break;
1557	case MSR_IA32_VMX_CR4_FIXED0:
1558		*pdata = msrs->cr4_fixed0;
1559		break;
1560	case MSR_IA32_VMX_CR4_FIXED1:
1561		*pdata = msrs->cr4_fixed1;
1562		break;
1563	case MSR_IA32_VMX_VMCS_ENUM:
1564		*pdata = msrs->vmcs_enum;
1565		break;
1566	case MSR_IA32_VMX_PROCBASED_CTLS2:
1567		*pdata = vmx_control_msr(
1568			msrs->secondary_ctls_low,
1569			msrs->secondary_ctls_high);
1570		break;
1571	case MSR_IA32_VMX_EPT_VPID_CAP:
1572		*pdata = msrs->ept_caps |
1573			((u64)msrs->vpid_caps << 32);
1574		break;
1575	case MSR_IA32_VMX_VMFUNC:
1576		*pdata = msrs->vmfunc_controls;
1577		break;
1578	default:
1579		return 1;
1580	}
1581
1582	return 0;
1583}
1584
1585/*
1586 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have
1587 * been modified by the L1 guest.  Note, "writable" in this context means
1588 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of
1589 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only"
1590 * VM-exit information fields (which are actually writable if the vCPU is
1591 * configured to support "VMWRITE to any supported field in the VMCS").
1592 */
1593static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
1594{
1595	struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1596	struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1597	struct shadow_vmcs_field field;
1598	unsigned long val;
1599	int i;
1600
1601	if (WARN_ON(!shadow_vmcs))
1602		return;
1603
1604	preempt_disable();
1605
1606	vmcs_load(shadow_vmcs);
1607
1608	for (i = 0; i < max_shadow_read_write_fields; i++) {
1609		field = shadow_read_write_fields[i];
1610		val = __vmcs_readl(field.encoding);
1611		vmcs12_write_any(vmcs12, field.encoding, field.offset, val);
1612	}
1613
1614	vmcs_clear(shadow_vmcs);
1615	vmcs_load(vmx->loaded_vmcs->vmcs);
1616
1617	preempt_enable();
1618}
1619
1620static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
1621{
1622	const struct shadow_vmcs_field *fields[] = {
1623		shadow_read_write_fields,
1624		shadow_read_only_fields
1625	};
1626	const int max_fields[] = {
1627		max_shadow_read_write_fields,
1628		max_shadow_read_only_fields
1629	};
1630	struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1631	struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1632	struct shadow_vmcs_field field;
1633	unsigned long val;
1634	int i, q;
1635
1636	if (WARN_ON(!shadow_vmcs))
1637		return;
1638
1639	vmcs_load(shadow_vmcs);
1640
1641	for (q = 0; q < ARRAY_SIZE(fields); q++) {
1642		for (i = 0; i < max_fields[q]; i++) {
1643			field = fields[q][i];
1644			val = vmcs12_read_any(vmcs12, field.encoding,
1645					      field.offset);
1646			__vmcs_writel(field.encoding, val);
1647		}
1648	}
1649
1650	vmcs_clear(shadow_vmcs);
1651	vmcs_load(vmx->loaded_vmcs->vmcs);
1652}
1653
1654static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields)
1655{
1656#ifdef CONFIG_KVM_HYPERV
1657	struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1658	struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx);
1659	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu);
1660
1661	/* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */
1662	vmcs12->tpr_threshold = evmcs->tpr_threshold;
1663	vmcs12->guest_rip = evmcs->guest_rip;
1664
1665	if (unlikely(!(hv_clean_fields &
1666		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) {
1667		hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page;
1668		hv_vcpu->nested.vm_id = evmcs->hv_vm_id;
1669		hv_vcpu->nested.vp_id = evmcs->hv_vp_id;
1670	}
1671
1672	if (unlikely(!(hv_clean_fields &
1673		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) {
1674		vmcs12->guest_rsp = evmcs->guest_rsp;
1675		vmcs12->guest_rflags = evmcs->guest_rflags;
1676		vmcs12->guest_interruptibility_info =
1677			evmcs->guest_interruptibility_info;
1678		/*
1679		 * Not present in struct vmcs12:
1680		 * vmcs12->guest_ssp = evmcs->guest_ssp;
1681		 */
1682	}
1683
1684	if (unlikely(!(hv_clean_fields &
1685		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) {
1686		vmcs12->cpu_based_vm_exec_control =
1687			evmcs->cpu_based_vm_exec_control;
1688	}
1689
1690	if (unlikely(!(hv_clean_fields &
1691		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) {
1692		vmcs12->exception_bitmap = evmcs->exception_bitmap;
1693	}
1694
1695	if (unlikely(!(hv_clean_fields &
1696		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) {
1697		vmcs12->vm_entry_controls = evmcs->vm_entry_controls;
1698	}
1699
1700	if (unlikely(!(hv_clean_fields &
1701		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) {
1702		vmcs12->vm_entry_intr_info_field =
1703			evmcs->vm_entry_intr_info_field;
1704		vmcs12->vm_entry_exception_error_code =
1705			evmcs->vm_entry_exception_error_code;
1706		vmcs12->vm_entry_instruction_len =
1707			evmcs->vm_entry_instruction_len;
1708	}
1709
1710	if (unlikely(!(hv_clean_fields &
1711		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) {
1712		vmcs12->host_ia32_pat = evmcs->host_ia32_pat;
1713		vmcs12->host_ia32_efer = evmcs->host_ia32_efer;
1714		vmcs12->host_cr0 = evmcs->host_cr0;
1715		vmcs12->host_cr3 = evmcs->host_cr3;
1716		vmcs12->host_cr4 = evmcs->host_cr4;
1717		vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp;
1718		vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip;
1719		vmcs12->host_rip = evmcs->host_rip;
1720		vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs;
1721		vmcs12->host_es_selector = evmcs->host_es_selector;
1722		vmcs12->host_cs_selector = evmcs->host_cs_selector;
1723		vmcs12->host_ss_selector = evmcs->host_ss_selector;
1724		vmcs12->host_ds_selector = evmcs->host_ds_selector;
1725		vmcs12->host_fs_selector = evmcs->host_fs_selector;
1726		vmcs12->host_gs_selector = evmcs->host_gs_selector;
1727		vmcs12->host_tr_selector = evmcs->host_tr_selector;
1728		vmcs12->host_ia32_perf_global_ctrl = evmcs->host_ia32_perf_global_ctrl;
1729		/*
1730		 * Not present in struct vmcs12:
1731		 * vmcs12->host_ia32_s_cet = evmcs->host_ia32_s_cet;
1732		 * vmcs12->host_ssp = evmcs->host_ssp;
1733		 * vmcs12->host_ia32_int_ssp_table_addr = evmcs->host_ia32_int_ssp_table_addr;
1734		 */
1735	}
1736
1737	if (unlikely(!(hv_clean_fields &
1738		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) {
1739		vmcs12->pin_based_vm_exec_control =
1740			evmcs->pin_based_vm_exec_control;
1741		vmcs12->vm_exit_controls = evmcs->vm_exit_controls;
1742		vmcs12->secondary_vm_exec_control =
1743			evmcs->secondary_vm_exec_control;
1744	}
1745
1746	if (unlikely(!(hv_clean_fields &
1747		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) {
1748		vmcs12->io_bitmap_a = evmcs->io_bitmap_a;
1749		vmcs12->io_bitmap_b = evmcs->io_bitmap_b;
1750	}
1751
1752	if (unlikely(!(hv_clean_fields &
1753		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) {
1754		vmcs12->msr_bitmap = evmcs->msr_bitmap;
1755	}
1756
1757	if (unlikely(!(hv_clean_fields &
1758		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) {
1759		vmcs12->guest_es_base = evmcs->guest_es_base;
1760		vmcs12->guest_cs_base = evmcs->guest_cs_base;
1761		vmcs12->guest_ss_base = evmcs->guest_ss_base;
1762		vmcs12->guest_ds_base = evmcs->guest_ds_base;
1763		vmcs12->guest_fs_base = evmcs->guest_fs_base;
1764		vmcs12->guest_gs_base = evmcs->guest_gs_base;
1765		vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base;
1766		vmcs12->guest_tr_base = evmcs->guest_tr_base;
1767		vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base;
1768		vmcs12->guest_idtr_base = evmcs->guest_idtr_base;
1769		vmcs12->guest_es_limit = evmcs->guest_es_limit;
1770		vmcs12->guest_cs_limit = evmcs->guest_cs_limit;
1771		vmcs12->guest_ss_limit = evmcs->guest_ss_limit;
1772		vmcs12->guest_ds_limit = evmcs->guest_ds_limit;
1773		vmcs12->guest_fs_limit = evmcs->guest_fs_limit;
1774		vmcs12->guest_gs_limit = evmcs->guest_gs_limit;
1775		vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit;
1776		vmcs12->guest_tr_limit = evmcs->guest_tr_limit;
1777		vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit;
1778		vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit;
1779		vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes;
1780		vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes;
1781		vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes;
1782		vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes;
1783		vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes;
1784		vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes;
1785		vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes;
1786		vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes;
1787		vmcs12->guest_es_selector = evmcs->guest_es_selector;
1788		vmcs12->guest_cs_selector = evmcs->guest_cs_selector;
1789		vmcs12->guest_ss_selector = evmcs->guest_ss_selector;
1790		vmcs12->guest_ds_selector = evmcs->guest_ds_selector;
1791		vmcs12->guest_fs_selector = evmcs->guest_fs_selector;
1792		vmcs12->guest_gs_selector = evmcs->guest_gs_selector;
1793		vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector;
1794		vmcs12->guest_tr_selector = evmcs->guest_tr_selector;
1795	}
1796
1797	if (unlikely(!(hv_clean_fields &
1798		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) {
1799		vmcs12->tsc_offset = evmcs->tsc_offset;
1800		vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr;
1801		vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap;
1802		vmcs12->encls_exiting_bitmap = evmcs->encls_exiting_bitmap;
1803		vmcs12->tsc_multiplier = evmcs->tsc_multiplier;
1804	}
1805
1806	if (unlikely(!(hv_clean_fields &
1807		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) {
1808		vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask;
1809		vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask;
1810		vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow;
1811		vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow;
1812		vmcs12->guest_cr0 = evmcs->guest_cr0;
1813		vmcs12->guest_cr3 = evmcs->guest_cr3;
1814		vmcs12->guest_cr4 = evmcs->guest_cr4;
1815		vmcs12->guest_dr7 = evmcs->guest_dr7;
1816	}
1817
1818	if (unlikely(!(hv_clean_fields &
1819		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) {
1820		vmcs12->host_fs_base = evmcs->host_fs_base;
1821		vmcs12->host_gs_base = evmcs->host_gs_base;
1822		vmcs12->host_tr_base = evmcs->host_tr_base;
1823		vmcs12->host_gdtr_base = evmcs->host_gdtr_base;
1824		vmcs12->host_idtr_base = evmcs->host_idtr_base;
1825		vmcs12->host_rsp = evmcs->host_rsp;
1826	}
1827
1828	if (unlikely(!(hv_clean_fields &
1829		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) {
1830		vmcs12->ept_pointer = evmcs->ept_pointer;
1831		vmcs12->virtual_processor_id = evmcs->virtual_processor_id;
1832	}
1833
1834	if (unlikely(!(hv_clean_fields &
1835		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) {
1836		vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer;
1837		vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl;
1838		vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat;
1839		vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer;
1840		vmcs12->guest_pdptr0 = evmcs->guest_pdptr0;
1841		vmcs12->guest_pdptr1 = evmcs->guest_pdptr1;
1842		vmcs12->guest_pdptr2 = evmcs->guest_pdptr2;
1843		vmcs12->guest_pdptr3 = evmcs->guest_pdptr3;
1844		vmcs12->guest_pending_dbg_exceptions =
1845			evmcs->guest_pending_dbg_exceptions;
1846		vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp;
1847		vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip;
1848		vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs;
1849		vmcs12->guest_activity_state = evmcs->guest_activity_state;
1850		vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs;
1851		vmcs12->guest_ia32_perf_global_ctrl = evmcs->guest_ia32_perf_global_ctrl;
1852		/*
1853		 * Not present in struct vmcs12:
1854		 * vmcs12->guest_ia32_s_cet = evmcs->guest_ia32_s_cet;
1855		 * vmcs12->guest_ia32_lbr_ctl = evmcs->guest_ia32_lbr_ctl;
1856		 * vmcs12->guest_ia32_int_ssp_table_addr = evmcs->guest_ia32_int_ssp_table_addr;
1857		 */
1858	}
1859
1860	/*
1861	 * Not used?
1862	 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr;
1863	 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr;
1864	 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr;
1865	 * vmcs12->page_fault_error_code_mask =
1866	 *		evmcs->page_fault_error_code_mask;
1867	 * vmcs12->page_fault_error_code_match =
1868	 *		evmcs->page_fault_error_code_match;
1869	 * vmcs12->cr3_target_count = evmcs->cr3_target_count;
1870	 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count;
1871	 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count;
1872	 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count;
1873	 */
1874
1875	/*
1876	 * Read only fields:
1877	 * vmcs12->guest_physical_address = evmcs->guest_physical_address;
1878	 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error;
1879	 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason;
1880	 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info;
1881	 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code;
1882	 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field;
1883	 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code;
1884	 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len;
1885	 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info;
1886	 * vmcs12->exit_qualification = evmcs->exit_qualification;
1887	 * vmcs12->guest_linear_address = evmcs->guest_linear_address;
1888	 *
1889	 * Not present in struct vmcs12:
1890	 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx;
1891	 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi;
1892	 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi;
1893	 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip;
1894	 */
1895
1896	return;
1897#else /* CONFIG_KVM_HYPERV */
1898	KVM_BUG_ON(1, vmx->vcpu.kvm);
1899#endif /* CONFIG_KVM_HYPERV */
1900}
1901
1902static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx)
1903{
1904#ifdef CONFIG_KVM_HYPERV
1905	struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1906	struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx);
1907
1908	/*
1909	 * Should not be changed by KVM:
1910	 *
1911	 * evmcs->host_es_selector = vmcs12->host_es_selector;
1912	 * evmcs->host_cs_selector = vmcs12->host_cs_selector;
1913	 * evmcs->host_ss_selector = vmcs12->host_ss_selector;
1914	 * evmcs->host_ds_selector = vmcs12->host_ds_selector;
1915	 * evmcs->host_fs_selector = vmcs12->host_fs_selector;
1916	 * evmcs->host_gs_selector = vmcs12->host_gs_selector;
1917	 * evmcs->host_tr_selector = vmcs12->host_tr_selector;
1918	 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat;
1919	 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer;
1920	 * evmcs->host_cr0 = vmcs12->host_cr0;
1921	 * evmcs->host_cr3 = vmcs12->host_cr3;
1922	 * evmcs->host_cr4 = vmcs12->host_cr4;
1923	 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp;
1924	 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip;
1925	 * evmcs->host_rip = vmcs12->host_rip;
1926	 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs;
1927	 * evmcs->host_fs_base = vmcs12->host_fs_base;
1928	 * evmcs->host_gs_base = vmcs12->host_gs_base;
1929	 * evmcs->host_tr_base = vmcs12->host_tr_base;
1930	 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base;
1931	 * evmcs->host_idtr_base = vmcs12->host_idtr_base;
1932	 * evmcs->host_rsp = vmcs12->host_rsp;
1933	 * sync_vmcs02_to_vmcs12() doesn't read these:
1934	 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a;
1935	 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b;
1936	 * evmcs->msr_bitmap = vmcs12->msr_bitmap;
1937	 * evmcs->ept_pointer = vmcs12->ept_pointer;
1938	 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap;
1939	 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr;
1940	 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr;
1941	 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr;
1942	 * evmcs->tpr_threshold = vmcs12->tpr_threshold;
1943	 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id;
1944	 * evmcs->exception_bitmap = vmcs12->exception_bitmap;
1945	 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer;
1946	 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control;
1947	 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls;
1948	 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control;
1949	 * evmcs->page_fault_error_code_mask =
1950	 *		vmcs12->page_fault_error_code_mask;
1951	 * evmcs->page_fault_error_code_match =
1952	 *		vmcs12->page_fault_error_code_match;
1953	 * evmcs->cr3_target_count = vmcs12->cr3_target_count;
1954	 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr;
1955	 * evmcs->tsc_offset = vmcs12->tsc_offset;
1956	 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl;
1957	 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask;
1958	 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask;
1959	 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow;
1960	 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow;
1961	 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count;
1962	 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count;
1963	 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count;
1964	 * evmcs->guest_ia32_perf_global_ctrl = vmcs12->guest_ia32_perf_global_ctrl;
1965	 * evmcs->host_ia32_perf_global_ctrl = vmcs12->host_ia32_perf_global_ctrl;
1966	 * evmcs->encls_exiting_bitmap = vmcs12->encls_exiting_bitmap;
1967	 * evmcs->tsc_multiplier = vmcs12->tsc_multiplier;
1968	 *
1969	 * Not present in struct vmcs12:
1970	 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx;
1971	 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi;
1972	 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi;
1973	 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip;
1974	 * evmcs->host_ia32_s_cet = vmcs12->host_ia32_s_cet;
1975	 * evmcs->host_ssp = vmcs12->host_ssp;
1976	 * evmcs->host_ia32_int_ssp_table_addr = vmcs12->host_ia32_int_ssp_table_addr;
1977	 * evmcs->guest_ia32_s_cet = vmcs12->guest_ia32_s_cet;
1978	 * evmcs->guest_ia32_lbr_ctl = vmcs12->guest_ia32_lbr_ctl;
1979	 * evmcs->guest_ia32_int_ssp_table_addr = vmcs12->guest_ia32_int_ssp_table_addr;
1980	 * evmcs->guest_ssp = vmcs12->guest_ssp;
1981	 */
1982
1983	evmcs->guest_es_selector = vmcs12->guest_es_selector;
1984	evmcs->guest_cs_selector = vmcs12->guest_cs_selector;
1985	evmcs->guest_ss_selector = vmcs12->guest_ss_selector;
1986	evmcs->guest_ds_selector = vmcs12->guest_ds_selector;
1987	evmcs->guest_fs_selector = vmcs12->guest_fs_selector;
1988	evmcs->guest_gs_selector = vmcs12->guest_gs_selector;
1989	evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector;
1990	evmcs->guest_tr_selector = vmcs12->guest_tr_selector;
1991
1992	evmcs->guest_es_limit = vmcs12->guest_es_limit;
1993	evmcs->guest_cs_limit = vmcs12->guest_cs_limit;
1994	evmcs->guest_ss_limit = vmcs12->guest_ss_limit;
1995	evmcs->guest_ds_limit = vmcs12->guest_ds_limit;
1996	evmcs->guest_fs_limit = vmcs12->guest_fs_limit;
1997	evmcs->guest_gs_limit = vmcs12->guest_gs_limit;
1998	evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit;
1999	evmcs->guest_tr_limit = vmcs12->guest_tr_limit;
2000	evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit;
2001	evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit;
2002
2003	evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes;
2004	evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes;
2005	evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes;
2006	evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes;
2007	evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes;
2008	evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes;
2009	evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes;
2010	evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes;
2011
2012	evmcs->guest_es_base = vmcs12->guest_es_base;
2013	evmcs->guest_cs_base = vmcs12->guest_cs_base;
2014	evmcs->guest_ss_base = vmcs12->guest_ss_base;
2015	evmcs->guest_ds_base = vmcs12->guest_ds_base;
2016	evmcs->guest_fs_base = vmcs12->guest_fs_base;
2017	evmcs->guest_gs_base = vmcs12->guest_gs_base;
2018	evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base;
2019	evmcs->guest_tr_base = vmcs12->guest_tr_base;
2020	evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base;
2021	evmcs->guest_idtr_base = vmcs12->guest_idtr_base;
2022
2023	evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat;
2024	evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer;
2025
2026	evmcs->guest_pdptr0 = vmcs12->guest_pdptr0;
2027	evmcs->guest_pdptr1 = vmcs12->guest_pdptr1;
2028	evmcs->guest_pdptr2 = vmcs12->guest_pdptr2;
2029	evmcs->guest_pdptr3 = vmcs12->guest_pdptr3;
2030
2031	evmcs->guest_pending_dbg_exceptions =
2032		vmcs12->guest_pending_dbg_exceptions;
2033	evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp;
2034	evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip;
2035
2036	evmcs->guest_activity_state = vmcs12->guest_activity_state;
2037	evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs;
2038
2039	evmcs->guest_cr0 = vmcs12->guest_cr0;
2040	evmcs->guest_cr3 = vmcs12->guest_cr3;
2041	evmcs->guest_cr4 = vmcs12->guest_cr4;
2042	evmcs->guest_dr7 = vmcs12->guest_dr7;
2043
2044	evmcs->guest_physical_address = vmcs12->guest_physical_address;
2045
2046	evmcs->vm_instruction_error = vmcs12->vm_instruction_error;
2047	evmcs->vm_exit_reason = vmcs12->vm_exit_reason;
2048	evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info;
2049	evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code;
2050	evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field;
2051	evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code;
2052	evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len;
2053	evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info;
2054
2055	evmcs->exit_qualification = vmcs12->exit_qualification;
2056
2057	evmcs->guest_linear_address = vmcs12->guest_linear_address;
2058	evmcs->guest_rsp = vmcs12->guest_rsp;
2059	evmcs->guest_rflags = vmcs12->guest_rflags;
2060
2061	evmcs->guest_interruptibility_info =
2062		vmcs12->guest_interruptibility_info;
2063	evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control;
2064	evmcs->vm_entry_controls = vmcs12->vm_entry_controls;
2065	evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field;
2066	evmcs->vm_entry_exception_error_code =
2067		vmcs12->vm_entry_exception_error_code;
2068	evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len;
2069
2070	evmcs->guest_rip = vmcs12->guest_rip;
2071
2072	evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs;
2073
2074	return;
2075#else /* CONFIG_KVM_HYPERV */
2076	KVM_BUG_ON(1, vmx->vcpu.kvm);
2077#endif /* CONFIG_KVM_HYPERV */
2078}
2079
2080/*
2081 * This is an equivalent of the nested hypervisor executing the vmptrld
2082 * instruction.
2083 */
2084static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld(
2085	struct kvm_vcpu *vcpu, bool from_launch)
2086{
2087#ifdef CONFIG_KVM_HYPERV
2088	struct vcpu_vmx *vmx = to_vmx(vcpu);
2089	bool evmcs_gpa_changed = false;
2090	u64 evmcs_gpa;
2091
2092	if (likely(!guest_cpuid_has_evmcs(vcpu)))
2093		return EVMPTRLD_DISABLED;
2094
2095	evmcs_gpa = nested_get_evmptr(vcpu);
2096	if (!evmptr_is_valid(evmcs_gpa)) {
2097		nested_release_evmcs(vcpu);
2098		return EVMPTRLD_DISABLED;
2099	}
2100
2101	if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) {
2102		vmx->nested.current_vmptr = INVALID_GPA;
2103
2104		nested_release_evmcs(vcpu);
2105
2106		if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa),
2107				 &vmx->nested.hv_evmcs_map))
2108			return EVMPTRLD_ERROR;
2109
2110		vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva;
2111
2112		/*
2113		 * Currently, KVM only supports eVMCS version 1
2114		 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this
2115		 * value to first u32 field of eVMCS which should specify eVMCS
2116		 * VersionNumber.
2117		 *
2118		 * Guest should be aware of supported eVMCS versions by host by
2119		 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is
2120		 * expected to set this CPUID leaf according to the value
2121		 * returned in vmcs_version from nested_enable_evmcs().
2122		 *
2123		 * However, it turns out that Microsoft Hyper-V fails to comply
2124		 * to their own invented interface: When Hyper-V use eVMCS, it
2125		 * just sets first u32 field of eVMCS to revision_id specified
2126		 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number
2127		 * which is one of the supported versions specified in
2128		 * CPUID.0x4000000A.EAX[0:15].
2129		 *
2130		 * To overcome Hyper-V bug, we accept here either a supported
2131		 * eVMCS version or VMCS12 revision_id as valid values for first
2132		 * u32 field of eVMCS.
2133		 */
2134		if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) &&
2135		    (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) {
2136			nested_release_evmcs(vcpu);
2137			return EVMPTRLD_VMFAIL;
2138		}
2139
2140		vmx->nested.hv_evmcs_vmptr = evmcs_gpa;
2141
2142		evmcs_gpa_changed = true;
2143		/*
2144		 * Unlike normal vmcs12, enlightened vmcs12 is not fully
2145		 * reloaded from guest's memory (read only fields, fields not
2146		 * present in struct hv_enlightened_vmcs, ...). Make sure there
2147		 * are no leftovers.
2148		 */
2149		if (from_launch) {
2150			struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
2151			memset(vmcs12, 0, sizeof(*vmcs12));
2152			vmcs12->hdr.revision_id = VMCS12_REVISION;
2153		}
2154
2155	}
2156
2157	/*
2158	 * Clean fields data can't be used on VMLAUNCH and when we switch
2159	 * between different L2 guests as KVM keeps a single VMCS12 per L1.
2160	 */
2161	if (from_launch || evmcs_gpa_changed) {
2162		vmx->nested.hv_evmcs->hv_clean_fields &=
2163			~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
2164
2165		vmx->nested.force_msr_bitmap_recalc = true;
2166	}
2167
2168	return EVMPTRLD_SUCCEEDED;
2169#else
2170	return EVMPTRLD_DISABLED;
2171#endif
2172}
2173
2174void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu)
2175{
2176	struct vcpu_vmx *vmx = to_vmx(vcpu);
2177
2178	if (nested_vmx_is_evmptr12_valid(vmx))
2179		copy_vmcs12_to_enlightened(vmx);
2180	else
2181		copy_vmcs12_to_shadow(vmx);
2182
2183	vmx->nested.need_vmcs12_to_shadow_sync = false;
2184}
2185
2186static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
2187{
2188	struct vcpu_vmx *vmx =
2189		container_of(timer, struct vcpu_vmx, nested.preemption_timer);
2190
2191	vmx->nested.preemption_timer_expired = true;
2192	kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
2193	kvm_vcpu_kick(&vmx->vcpu);
2194
2195	return HRTIMER_NORESTART;
2196}
2197
2198static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu)
2199{
2200	struct vcpu_vmx *vmx = to_vmx(vcpu);
2201	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
2202
2203	u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >>
2204			    VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
2205
2206	if (!vmx->nested.has_preemption_timer_deadline) {
2207		vmx->nested.preemption_timer_deadline =
2208			vmcs12->vmx_preemption_timer_value + l1_scaled_tsc;
2209		vmx->nested.has_preemption_timer_deadline = true;
2210	}
2211	return vmx->nested.preemption_timer_deadline - l1_scaled_tsc;
2212}
2213
2214static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu,
2215					u64 preemption_timeout)
2216{
2217	struct vcpu_vmx *vmx = to_vmx(vcpu);
2218
2219	/*
2220	 * A timer value of zero is architecturally guaranteed to cause
2221	 * a VMExit prior to executing any instructions in the guest.
2222	 */
2223	if (preemption_timeout == 0) {
2224		vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
2225		return;
2226	}
2227
2228	if (vcpu->arch.virtual_tsc_khz == 0)
2229		return;
2230
2231	preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
2232	preemption_timeout *= 1000000;
2233	do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
2234	hrtimer_start(&vmx->nested.preemption_timer,
2235		      ktime_add_ns(ktime_get(), preemption_timeout),
2236		      HRTIMER_MODE_ABS_PINNED);
2237}
2238
2239static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2240{
2241	if (vmx->nested.nested_run_pending &&
2242	    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
2243		return vmcs12->guest_ia32_efer;
2244	else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
2245		return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME);
2246	else
2247		return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME);
2248}
2249
2250static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx)
2251{
2252	struct kvm *kvm = vmx->vcpu.kvm;
2253
2254	/*
2255	 * If vmcs02 hasn't been initialized, set the constant vmcs02 state
2256	 * according to L0's settings (vmcs12 is irrelevant here).  Host
2257	 * fields that come from L0 and are not constant, e.g. HOST_CR3,
2258	 * will be set as needed prior to VMLAUNCH/VMRESUME.
2259	 */
2260	if (vmx->nested.vmcs02_initialized)
2261		return;
2262	vmx->nested.vmcs02_initialized = true;
2263
2264	/*
2265	 * We don't care what the EPTP value is we just need to guarantee
2266	 * it's valid so we don't get a false positive when doing early
2267	 * consistency checks.
2268	 */
2269	if (enable_ept && nested_early_check)
2270		vmcs_write64(EPT_POINTER,
2271			     construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL));
2272
2273	if (vmx->ve_info)
2274		vmcs_write64(VE_INFORMATION_ADDRESS, __pa(vmx->ve_info));
2275
2276	/* All VMFUNCs are currently emulated through L0 vmexits.  */
2277	if (cpu_has_vmx_vmfunc())
2278		vmcs_write64(VM_FUNCTION_CONTROL, 0);
2279
2280	if (cpu_has_vmx_posted_intr())
2281		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
2282
2283	if (cpu_has_vmx_msr_bitmap())
2284		vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
2285
2286	/*
2287	 * PML is emulated for L2, but never enabled in hardware as the MMU
2288	 * handles A/D emulation.  Disabling PML for L2 also avoids having to
2289	 * deal with filtering out L2 GPAs from the buffer.
2290	 */
2291	if (enable_pml) {
2292		vmcs_write64(PML_ADDRESS, 0);
2293		vmcs_write16(GUEST_PML_INDEX, -1);
2294	}
2295
2296	if (cpu_has_vmx_encls_vmexit())
2297		vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA);
2298
2299	if (kvm_notify_vmexit_enabled(kvm))
2300		vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window);
2301
2302	/*
2303	 * Set the MSR load/store lists to match L0's settings.  Only the
2304	 * addresses are constant (for vmcs02), the counts can change based
2305	 * on L2's behavior, e.g. switching to/from long mode.
2306	 */
2307	vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val));
2308	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
2309	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
2310
2311	vmx_set_constant_host_state(vmx);
2312}
2313
2314static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx,
2315				      struct vmcs12 *vmcs12)
2316{
2317	prepare_vmcs02_constant_state(vmx);
2318
2319	vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA);
2320
2321	/*
2322	 * If VPID is disabled, then guest TLB accesses use VPID=0, i.e. the
2323	 * same VPID as the host.  Emulate this behavior by using vpid01 for L2
2324	 * if VPID is disabled in vmcs12.  Note, if VPID is disabled, VM-Enter
2325	 * and VM-Exit are architecturally required to flush VPID=0, but *only*
2326	 * VPID=0.  I.e. using vpid02 would be ok (so long as KVM emulates the
2327	 * required flushes), but doing so would cause KVM to over-flush.  E.g.
2328	 * if L1 runs L2 X with VPID12=1, then runs L2 Y with VPID12 disabled,
2329	 * and then runs L2 X again, then KVM can and should retain TLB entries
2330	 * for VPID12=1.
2331	 */
2332	if (enable_vpid) {
2333		if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02)
2334			vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
2335		else
2336			vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2337	}
2338}
2339
2340static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01,
2341				 struct vmcs12 *vmcs12)
2342{
2343	u32 exec_control;
2344	u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12);
2345
2346	if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx))
2347		prepare_vmcs02_early_rare(vmx, vmcs12);
2348
2349	/*
2350	 * PIN CONTROLS
2351	 */
2352	exec_control = __pin_controls_get(vmcs01);
2353	exec_control |= (vmcs12->pin_based_vm_exec_control &
2354			 ~PIN_BASED_VMX_PREEMPTION_TIMER);
2355
2356	/* Posted interrupts setting is only taken from vmcs12.  */
2357	vmx->nested.pi_pending = false;
2358	if (nested_cpu_has_posted_intr(vmcs12)) {
2359		vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
2360	} else {
2361		vmx->nested.posted_intr_nv = -1;
2362		exec_control &= ~PIN_BASED_POSTED_INTR;
2363	}
2364	pin_controls_set(vmx, exec_control);
2365
2366	/*
2367	 * EXEC CONTROLS
2368	 */
2369	exec_control = __exec_controls_get(vmcs01); /* L0's desires */
2370	exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING;
2371	exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING;
2372	exec_control &= ~CPU_BASED_TPR_SHADOW;
2373	exec_control |= vmcs12->cpu_based_vm_exec_control;
2374
2375	vmx->nested.l1_tpr_threshold = -1;
2376	if (exec_control & CPU_BASED_TPR_SHADOW)
2377		vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
2378#ifdef CONFIG_X86_64
2379	else
2380		exec_control |= CPU_BASED_CR8_LOAD_EXITING |
2381				CPU_BASED_CR8_STORE_EXITING;
2382#endif
2383
2384	/*
2385	 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed
2386	 * for I/O port accesses.
2387	 */
2388	exec_control |= CPU_BASED_UNCOND_IO_EXITING;
2389	exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
2390
2391	/*
2392	 * This bit will be computed in nested_get_vmcs12_pages, because
2393	 * we do not have access to L1's MSR bitmap yet.  For now, keep
2394	 * the same bit as before, hoping to avoid multiple VMWRITEs that
2395	 * only set/clear this bit.
2396	 */
2397	exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;
2398	exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS;
2399
2400	exec_controls_set(vmx, exec_control);
2401
2402	/*
2403	 * SECONDARY EXEC CONTROLS
2404	 */
2405	if (cpu_has_secondary_exec_ctrls()) {
2406		exec_control = __secondary_exec_controls_get(vmcs01);
2407
2408		/* Take the following fields only from vmcs12 */
2409		exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2410				  SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2411				  SECONDARY_EXEC_ENABLE_INVPCID |
2412				  SECONDARY_EXEC_ENABLE_RDTSCP |
2413				  SECONDARY_EXEC_ENABLE_XSAVES |
2414				  SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
2415				  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2416				  SECONDARY_EXEC_APIC_REGISTER_VIRT |
2417				  SECONDARY_EXEC_ENABLE_VMFUNC |
2418				  SECONDARY_EXEC_DESC);
2419
2420		if (nested_cpu_has(vmcs12,
2421				   CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
2422			exec_control |= vmcs12->secondary_vm_exec_control;
2423
2424		/* PML is emulated and never enabled in hardware for L2. */
2425		exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
2426
2427		/* VMCS shadowing for L2 is emulated for now */
2428		exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
2429
2430		/*
2431		 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4()
2432		 * will not have to rewrite the controls just for this bit.
2433		 */
2434		if (vmx_umip_emulated() && (vmcs12->guest_cr4 & X86_CR4_UMIP))
2435			exec_control |= SECONDARY_EXEC_DESC;
2436
2437		if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
2438			vmcs_write16(GUEST_INTR_STATUS,
2439				vmcs12->guest_intr_status);
2440
2441		if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
2442		    exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
2443
2444		if (exec_control & SECONDARY_EXEC_ENCLS_EXITING)
2445			vmx_write_encls_bitmap(&vmx->vcpu, vmcs12);
2446
2447		secondary_exec_controls_set(vmx, exec_control);
2448	}
2449
2450	/*
2451	 * ENTRY CONTROLS
2452	 *
2453	 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE
2454	 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate
2455	 * on the related bits (if supported by the CPU) in the hope that
2456	 * we can avoid VMWrites during vmx_set_efer().
2457	 *
2458	 * Similarly, take vmcs01's PERF_GLOBAL_CTRL in the hope that if KVM is
2459	 * loading PERF_GLOBAL_CTRL via the VMCS for L1, then KVM will want to
2460	 * do the same for L2.
2461	 */
2462	exec_control = __vm_entry_controls_get(vmcs01);
2463	exec_control |= (vmcs12->vm_entry_controls &
2464			 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL);
2465	exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER);
2466	if (cpu_has_load_ia32_efer()) {
2467		if (guest_efer & EFER_LMA)
2468			exec_control |= VM_ENTRY_IA32E_MODE;
2469		if (guest_efer != kvm_host.efer)
2470			exec_control |= VM_ENTRY_LOAD_IA32_EFER;
2471	}
2472	vm_entry_controls_set(vmx, exec_control);
2473
2474	/*
2475	 * EXIT CONTROLS
2476	 *
2477	 * L2->L1 exit controls are emulated - the hardware exit is to L0 so
2478	 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
2479	 * bits may be modified by vmx_set_efer() in prepare_vmcs02().
2480	 */
2481	exec_control = __vm_exit_controls_get(vmcs01);
2482	if (cpu_has_load_ia32_efer() && guest_efer != kvm_host.efer)
2483		exec_control |= VM_EXIT_LOAD_IA32_EFER;
2484	else
2485		exec_control &= ~VM_EXIT_LOAD_IA32_EFER;
2486	vm_exit_controls_set(vmx, exec_control);
2487
2488	/*
2489	 * Interrupt/Exception Fields
2490	 */
2491	if (vmx->nested.nested_run_pending) {
2492		vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2493			     vmcs12->vm_entry_intr_info_field);
2494		vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
2495			     vmcs12->vm_entry_exception_error_code);
2496		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2497			     vmcs12->vm_entry_instruction_len);
2498		vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
2499			     vmcs12->guest_interruptibility_info);
2500		vmx->loaded_vmcs->nmi_known_unmasked =
2501			!(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI);
2502	} else {
2503		vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
2504	}
2505}
2506
2507static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2508{
2509	struct hv_enlightened_vmcs *hv_evmcs = nested_vmx_evmcs(vmx);
2510
2511	if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
2512			   HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) {
2513
2514		vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
2515		vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
2516		vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
2517		vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
2518		vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
2519		vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
2520		vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
2521		vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
2522		vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
2523		vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
2524		vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
2525		vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
2526		vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
2527		vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
2528		vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
2529		vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
2530		vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
2531		vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
2532		vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
2533		vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
2534		vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
2535		vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
2536		vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
2537		vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
2538		vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
2539		vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
2540		vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
2541		vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
2542		vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
2543		vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
2544		vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
2545		vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
2546		vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
2547		vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
2548		vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
2549		vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
2550
2551		vmx_segment_cache_clear(vmx);
2552	}
2553
2554	if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
2555			   HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) {
2556		vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
2557		vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
2558			    vmcs12->guest_pending_dbg_exceptions);
2559		vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
2560		vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
2561
2562		/*
2563		 * L1 may access the L2's PDPTR, so save them to construct
2564		 * vmcs12
2565		 */
2566		if (enable_ept) {
2567			vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
2568			vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
2569			vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
2570			vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
2571		}
2572
2573		if (kvm_mpx_supported() && vmx->nested.nested_run_pending &&
2574		    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
2575			vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
2576	}
2577
2578	if (nested_cpu_has_xsaves(vmcs12))
2579		vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
2580
2581	/*
2582	 * Whether page-faults are trapped is determined by a combination of
2583	 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.  If L0
2584	 * doesn't care about page faults then we should set all of these to
2585	 * L1's desires. However, if L0 does care about (some) page faults, it
2586	 * is not easy (if at all possible?) to merge L0 and L1's desires, we
2587	 * simply ask to exit on each and every L2 page fault. This is done by
2588	 * setting MASK=MATCH=0 and (see below) EB.PF=1.
2589	 * Note that below we don't need special code to set EB.PF beyond the
2590	 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
2591	 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
2592	 * !enable_ept, EB.PF is 1, so the "or" will always be 1.
2593	 */
2594	if (vmx_need_pf_intercept(&vmx->vcpu)) {
2595		/*
2596		 * TODO: if both L0 and L1 need the same MASK and MATCH,
2597		 * go ahead and use it?
2598		 */
2599		vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
2600		vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
2601	} else {
2602		vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask);
2603		vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match);
2604	}
2605
2606	if (cpu_has_vmx_apicv()) {
2607		vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0);
2608		vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1);
2609		vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2);
2610		vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3);
2611	}
2612
2613	/*
2614	 * Make sure the msr_autostore list is up to date before we set the
2615	 * count in the vmcs02.
2616	 */
2617	prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC);
2618
2619	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr);
2620	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
2621	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2622
2623	set_cr4_guest_host_mask(vmx);
2624}
2625
2626/*
2627 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
2628 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
2629 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
2630 * guest in a way that will both be appropriate to L1's requests, and our
2631 * needs. In addition to modifying the active vmcs (which is vmcs02), this
2632 * function also has additional necessary side-effects, like setting various
2633 * vcpu->arch fields.
2634 * Returns 0 on success, 1 on failure. Invalid state exit qualification code
2635 * is assigned to entry_failure_code on failure.
2636 */
2637static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
2638			  bool from_vmentry,
2639			  enum vm_entry_failure_code *entry_failure_code)
2640{
2641	struct vcpu_vmx *vmx = to_vmx(vcpu);
2642	struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx);
2643	bool load_guest_pdptrs_vmcs12 = false;
2644
2645	if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) {
2646		prepare_vmcs02_rare(vmx, vmcs12);
2647		vmx->nested.dirty_vmcs12 = false;
2648
2649		load_guest_pdptrs_vmcs12 = !nested_vmx_is_evmptr12_valid(vmx) ||
2650			!(evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1);
2651	}
2652
2653	if (vmx->nested.nested_run_pending &&
2654	    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
2655		kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
2656		vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
2657	} else {
2658		kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
2659		vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.pre_vmenter_debugctl);
2660	}
2661	if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending ||
2662	    !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)))
2663		vmcs_write64(GUEST_BNDCFGS, vmx->nested.pre_vmenter_bndcfgs);
2664	vmx_set_rflags(vcpu, vmcs12->guest_rflags);
2665
2666	/* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
2667	 * bitwise-or of what L1 wants to trap for L2, and what we want to
2668	 * trap. Note that CR0.TS also needs updating - we do this later.
2669	 */
2670	vmx_update_exception_bitmap(vcpu);
2671	vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
2672	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
2673
2674	if (vmx->nested.nested_run_pending &&
2675	    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
2676		vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
2677		vcpu->arch.pat = vmcs12->guest_ia32_pat;
2678	} else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2679		vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
2680	}
2681
2682	vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
2683			vcpu->arch.l1_tsc_offset,
2684			vmx_get_l2_tsc_offset(vcpu),
2685			vmx_get_l2_tsc_multiplier(vcpu));
2686
2687	vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
2688			vcpu->arch.l1_tsc_scaling_ratio,
2689			vmx_get_l2_tsc_multiplier(vcpu));
2690
2691	vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
2692	if (kvm_caps.has_tsc_control)
2693		vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio);
2694
2695	nested_vmx_transition_tlb_flush(vcpu, vmcs12, true);
2696
2697	if (nested_cpu_has_ept(vmcs12))
2698		nested_ept_init_mmu_context(vcpu);
2699
2700	/*
2701	 * Override the CR0/CR4 read shadows after setting the effective guest
2702	 * CR0/CR4.  The common helpers also set the shadows, but they don't
2703	 * account for vmcs12's cr0/4_guest_host_mask.
2704	 */
2705	vmx_set_cr0(vcpu, vmcs12->guest_cr0);
2706	vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
2707
2708	vmx_set_cr4(vcpu, vmcs12->guest_cr4);
2709	vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
2710
2711	vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12);
2712	/* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
2713	vmx_set_efer(vcpu, vcpu->arch.efer);
2714
2715	/*
2716	 * Guest state is invalid and unrestricted guest is disabled,
2717	 * which means L1 attempted VMEntry to L2 with invalid state.
2718	 * Fail the VMEntry.
2719	 *
2720	 * However when force loading the guest state (SMM exit or
2721	 * loading nested state after migration, it is possible to
2722	 * have invalid guest state now, which will be later fixed by
2723	 * restoring L2 register state
2724	 */
2725	if (CC(from_vmentry && !vmx_guest_state_valid(vcpu))) {
2726		*entry_failure_code = ENTRY_FAIL_DEFAULT;
2727		return -EINVAL;
2728	}
2729
2730	/* Shadow page tables on either EPT or shadow page tables. */
2731	if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12),
2732				from_vmentry, entry_failure_code))
2733		return -EINVAL;
2734
2735	/*
2736	 * Immediately write vmcs02.GUEST_CR3.  It will be propagated to vmcs12
2737	 * on nested VM-Exit, which can occur without actually running L2 and
2738	 * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with
2739	 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the
2740	 * transition to HLT instead of running L2.
2741	 */
2742	if (enable_ept)
2743		vmcs_writel(GUEST_CR3, vmcs12->guest_cr3);
2744
2745	/* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */
2746	if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) &&
2747	    is_pae_paging(vcpu)) {
2748		vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
2749		vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
2750		vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
2751		vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
2752	}
2753
2754	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) &&
2755	    kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)) &&
2756	    WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL,
2757				     vmcs12->guest_ia32_perf_global_ctrl))) {
2758		*entry_failure_code = ENTRY_FAIL_DEFAULT;
2759		return -EINVAL;
2760	}
2761
2762	kvm_rsp_write(vcpu, vmcs12->guest_rsp);
2763	kvm_rip_write(vcpu, vmcs12->guest_rip);
2764
2765	/*
2766	 * It was observed that genuine Hyper-V running in L1 doesn't reset
2767	 * 'hv_clean_fields' by itself, it only sets the corresponding dirty
2768	 * bits when it changes a field in eVMCS. Mark all fields as clean
2769	 * here.
2770	 */
2771	if (nested_vmx_is_evmptr12_valid(vmx))
2772		evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
2773
2774	return 0;
2775}
2776
2777static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)
2778{
2779	if (CC(!nested_cpu_has_nmi_exiting(vmcs12) &&
2780	       nested_cpu_has_virtual_nmis(vmcs12)))
2781		return -EINVAL;
2782
2783	if (CC(!nested_cpu_has_virtual_nmis(vmcs12) &&
2784	       nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING)))
2785		return -EINVAL;
2786
2787	return 0;
2788}
2789
2790static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp)
2791{
2792	struct vcpu_vmx *vmx = to_vmx(vcpu);
2793
2794	/* Check for memory type validity */
2795	switch (new_eptp & VMX_EPTP_MT_MASK) {
2796	case VMX_EPTP_MT_UC:
2797		if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)))
2798			return false;
2799		break;
2800	case VMX_EPTP_MT_WB:
2801		if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)))
2802			return false;
2803		break;
2804	default:
2805		return false;
2806	}
2807
2808	/* Page-walk levels validity. */
2809	switch (new_eptp & VMX_EPTP_PWL_MASK) {
2810	case VMX_EPTP_PWL_5:
2811		if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT)))
2812			return false;
2813		break;
2814	case VMX_EPTP_PWL_4:
2815		if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT)))
2816			return false;
2817		break;
2818	default:
2819		return false;
2820	}
2821
2822	/* Reserved bits should not be set */
2823	if (CC(!kvm_vcpu_is_legal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f)))
2824		return false;
2825
2826	/* AD, if set, should be supported */
2827	if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) {
2828		if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)))
2829			return false;
2830	}
2831
2832	return true;
2833}
2834
2835/*
2836 * Checks related to VM-Execution Control Fields
2837 */
2838static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu,
2839                                              struct vmcs12 *vmcs12)
2840{
2841	struct vcpu_vmx *vmx = to_vmx(vcpu);
2842
2843	if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control,
2844				   vmx->nested.msrs.pinbased_ctls_low,
2845				   vmx->nested.msrs.pinbased_ctls_high)) ||
2846	    CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
2847				   vmx->nested.msrs.procbased_ctls_low,
2848				   vmx->nested.msrs.procbased_ctls_high)))
2849		return -EINVAL;
2850
2851	if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
2852	    CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control,
2853				   vmx->nested.msrs.secondary_ctls_low,
2854				   vmx->nested.msrs.secondary_ctls_high)))
2855		return -EINVAL;
2856
2857	if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) ||
2858	    nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) ||
2859	    nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) ||
2860	    nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) ||
2861	    nested_vmx_check_apic_access_controls(vcpu, vmcs12) ||
2862	    nested_vmx_check_apicv_controls(vcpu, vmcs12) ||
2863	    nested_vmx_check_nmi_controls(vmcs12) ||
2864	    nested_vmx_check_pml_controls(vcpu, vmcs12) ||
2865	    nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) ||
2866	    nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) ||
2867	    nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) ||
2868	    CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))
2869		return -EINVAL;
2870
2871	if (!nested_cpu_has_preemption_timer(vmcs12) &&
2872	    nested_cpu_has_save_preemption_timer(vmcs12))
2873		return -EINVAL;
2874
2875	if (nested_cpu_has_ept(vmcs12) &&
2876	    CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer)))
2877		return -EINVAL;
2878
2879	if (nested_cpu_has_vmfunc(vmcs12)) {
2880		if (CC(vmcs12->vm_function_control &
2881		       ~vmx->nested.msrs.vmfunc_controls))
2882			return -EINVAL;
2883
2884		if (nested_cpu_has_eptp_switching(vmcs12)) {
2885			if (CC(!nested_cpu_has_ept(vmcs12)) ||
2886			    CC(!page_address_valid(vcpu, vmcs12->eptp_list_address)))
2887				return -EINVAL;
2888		}
2889	}
2890
2891	return 0;
2892}
2893
2894/*
2895 * Checks related to VM-Exit Control Fields
2896 */
2897static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu,
2898                                         struct vmcs12 *vmcs12)
2899{
2900	struct vcpu_vmx *vmx = to_vmx(vcpu);
2901
2902	if (CC(!vmx_control_verify(vmcs12->vm_exit_controls,
2903				    vmx->nested.msrs.exit_ctls_low,
2904				    vmx->nested.msrs.exit_ctls_high)) ||
2905	    CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)))
2906		return -EINVAL;
2907
2908	return 0;
2909}
2910
2911/*
2912 * Checks related to VM-Entry Control Fields
2913 */
2914static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu,
2915					  struct vmcs12 *vmcs12)
2916{
2917	struct vcpu_vmx *vmx = to_vmx(vcpu);
2918
2919	if (CC(!vmx_control_verify(vmcs12->vm_entry_controls,
2920				    vmx->nested.msrs.entry_ctls_low,
2921				    vmx->nested.msrs.entry_ctls_high)))
2922		return -EINVAL;
2923
2924	/*
2925	 * From the Intel SDM, volume 3:
2926	 * Fields relevant to VM-entry event injection must be set properly.
2927	 * These fields are the VM-entry interruption-information field, the
2928	 * VM-entry exception error code, and the VM-entry instruction length.
2929	 */
2930	if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) {
2931		u32 intr_info = vmcs12->vm_entry_intr_info_field;
2932		u8 vector = intr_info & INTR_INFO_VECTOR_MASK;
2933		u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK;
2934		bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK;
2935		bool should_have_error_code;
2936		bool urg = nested_cpu_has2(vmcs12,
2937					   SECONDARY_EXEC_UNRESTRICTED_GUEST);
2938		bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE;
2939
2940		/* VM-entry interruption-info field: interruption type */
2941		if (CC(intr_type == INTR_TYPE_RESERVED) ||
2942		    CC(intr_type == INTR_TYPE_OTHER_EVENT &&
2943		       !nested_cpu_supports_monitor_trap_flag(vcpu)))
2944			return -EINVAL;
2945
2946		/* VM-entry interruption-info field: vector */
2947		if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) ||
2948		    CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) ||
2949		    CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
2950			return -EINVAL;
2951
2952		/* VM-entry interruption-info field: deliver error code */
2953		should_have_error_code =
2954			intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&
2955			x86_exception_has_error_code(vector);
2956		if (CC(has_error_code != should_have_error_code))
2957			return -EINVAL;
2958
2959		/* VM-entry exception error code */
2960		if (CC(has_error_code &&
2961		       vmcs12->vm_entry_exception_error_code & GENMASK(31, 16)))
2962			return -EINVAL;
2963
2964		/* VM-entry interruption-info field: reserved bits */
2965		if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK))
2966			return -EINVAL;
2967
2968		/* VM-entry instruction length */
2969		switch (intr_type) {
2970		case INTR_TYPE_SOFT_EXCEPTION:
2971		case INTR_TYPE_SOFT_INTR:
2972		case INTR_TYPE_PRIV_SW_EXCEPTION:
2973			if (CC(vmcs12->vm_entry_instruction_len > 15) ||
2974			    CC(vmcs12->vm_entry_instruction_len == 0 &&
2975			    CC(!nested_cpu_has_zero_length_injection(vcpu))))
2976				return -EINVAL;
2977		}
2978	}
2979
2980	if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12))
2981		return -EINVAL;
2982
2983	return 0;
2984}
2985
2986static int nested_vmx_check_controls(struct kvm_vcpu *vcpu,
2987				     struct vmcs12 *vmcs12)
2988{
2989	if (nested_check_vm_execution_controls(vcpu, vmcs12) ||
2990	    nested_check_vm_exit_controls(vcpu, vmcs12) ||
2991	    nested_check_vm_entry_controls(vcpu, vmcs12))
2992		return -EINVAL;
2993
2994#ifdef CONFIG_KVM_HYPERV
2995	if (guest_cpuid_has_evmcs(vcpu))
2996		return nested_evmcs_check_controls(vmcs12);
2997#endif
2998
2999	return 0;
3000}
3001
3002static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu,
3003				       struct vmcs12 *vmcs12)
3004{
3005#ifdef CONFIG_X86_64
3006	if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) !=
3007		!!(vcpu->arch.efer & EFER_LMA)))
3008		return -EINVAL;
3009#endif
3010	return 0;
3011}
3012
3013static bool is_l1_noncanonical_address_on_vmexit(u64 la, struct vmcs12 *vmcs12)
3014{
3015	/*
3016	 * Check that the given linear address is canonical after a VM exit
3017	 * from L2, based on HOST_CR4.LA57 value that will be loaded for L1.
3018	 */
3019	u8 l1_address_bits_on_exit = (vmcs12->host_cr4 & X86_CR4_LA57) ? 57 : 48;
3020
3021	return !__is_canonical_address(la, l1_address_bits_on_exit);
3022}
3023
3024static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu,
3025				       struct vmcs12 *vmcs12)
3026{
3027	bool ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE);
3028
3029	if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) ||
3030	    CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) ||
3031	    CC(!kvm_vcpu_is_legal_cr3(vcpu, vmcs12->host_cr3)))
3032		return -EINVAL;
3033
3034	if (CC(is_noncanonical_msr_address(vmcs12->host_ia32_sysenter_esp, vcpu)) ||
3035	    CC(is_noncanonical_msr_address(vmcs12->host_ia32_sysenter_eip, vcpu)))
3036		return -EINVAL;
3037
3038	if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) &&
3039	    CC(!kvm_pat_valid(vmcs12->host_ia32_pat)))
3040		return -EINVAL;
3041
3042	if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) &&
3043	    CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu),
3044					   vmcs12->host_ia32_perf_global_ctrl)))
3045		return -EINVAL;
3046
3047	if (ia32e) {
3048		if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE)))
3049			return -EINVAL;
3050	} else {
3051		if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) ||
3052		    CC(vmcs12->host_cr4 & X86_CR4_PCIDE) ||
3053		    CC((vmcs12->host_rip) >> 32))
3054			return -EINVAL;
3055	}
3056
3057	if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
3058	    CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
3059	    CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
3060	    CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
3061	    CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
3062	    CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
3063	    CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
3064	    CC(vmcs12->host_cs_selector == 0) ||
3065	    CC(vmcs12->host_tr_selector == 0) ||
3066	    CC(vmcs12->host_ss_selector == 0 && !ia32e))
3067		return -EINVAL;
3068
3069	if (CC(is_noncanonical_base_address(vmcs12->host_fs_base, vcpu)) ||
3070	    CC(is_noncanonical_base_address(vmcs12->host_gs_base, vcpu)) ||
3071	    CC(is_noncanonical_base_address(vmcs12->host_gdtr_base, vcpu)) ||
3072	    CC(is_noncanonical_base_address(vmcs12->host_idtr_base, vcpu)) ||
3073	    CC(is_noncanonical_base_address(vmcs12->host_tr_base, vcpu)) ||
3074	    CC(is_l1_noncanonical_address_on_vmexit(vmcs12->host_rip, vmcs12)))
3075		return -EINVAL;
3076
3077	/*
3078	 * If the load IA32_EFER VM-exit control is 1, bits reserved in the
3079	 * IA32_EFER MSR must be 0 in the field for that register. In addition,
3080	 * the values of the LMA and LME bits in the field must each be that of
3081	 * the host address-space size VM-exit control.
3082	 */
3083	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
3084		if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) ||
3085		    CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) ||
3086		    CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)))
3087			return -EINVAL;
3088	}
3089
3090	return 0;
3091}
3092
3093static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu,
3094					  struct vmcs12 *vmcs12)
3095{
3096	struct vcpu_vmx *vmx = to_vmx(vcpu);
3097	struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache;
3098	struct vmcs_hdr hdr;
3099
3100	if (vmcs12->vmcs_link_pointer == INVALID_GPA)
3101		return 0;
3102
3103	if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)))
3104		return -EINVAL;
3105
3106	if (ghc->gpa != vmcs12->vmcs_link_pointer &&
3107	    CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc,
3108					 vmcs12->vmcs_link_pointer, VMCS12_SIZE)))
3109                return -EINVAL;
3110
3111	if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr,
3112					    offsetof(struct vmcs12, hdr),
3113					    sizeof(hdr))))
3114		return -EINVAL;
3115
3116	if (CC(hdr.revision_id != VMCS12_REVISION) ||
3117	    CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)))
3118		return -EINVAL;
3119
3120	return 0;
3121}
3122
3123/*
3124 * Checks related to Guest Non-register State
3125 */
3126static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12)
3127{
3128	if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
3129	       vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT &&
3130	       vmcs12->guest_activity_state != GUEST_ACTIVITY_WAIT_SIPI))
3131		return -EINVAL;
3132
3133	return 0;
3134}
3135
3136static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu,
3137					struct vmcs12 *vmcs12,
3138					enum vm_entry_failure_code *entry_failure_code)
3139{
3140	bool ia32e = !!(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE);
3141
3142	*entry_failure_code = ENTRY_FAIL_DEFAULT;
3143
3144	if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) ||
3145	    CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)))
3146		return -EINVAL;
3147
3148	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) &&
3149	    CC(!kvm_dr7_valid(vmcs12->guest_dr7)))
3150		return -EINVAL;
3151
3152	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) &&
3153	    CC(!kvm_pat_valid(vmcs12->guest_ia32_pat)))
3154		return -EINVAL;
3155
3156	if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {
3157		*entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR;
3158		return -EINVAL;
3159	}
3160
3161	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) &&
3162	    CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu),
3163					   vmcs12->guest_ia32_perf_global_ctrl)))
3164		return -EINVAL;
3165
3166	if (CC((vmcs12->guest_cr0 & (X86_CR0_PG | X86_CR0_PE)) == X86_CR0_PG))
3167		return -EINVAL;
3168
3169	if (CC(ia32e && !(vmcs12->guest_cr4 & X86_CR4_PAE)) ||
3170	    CC(ia32e && !(vmcs12->guest_cr0 & X86_CR0_PG)))
3171		return -EINVAL;
3172
3173	/*
3174	 * If the load IA32_EFER VM-entry control is 1, the following checks
3175	 * are performed on the field for the IA32_EFER MSR:
3176	 * - Bits reserved in the IA32_EFER MSR must be 0.
3177	 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
3178	 *   the IA-32e mode guest VM-exit control. It must also be identical
3179	 *   to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
3180	 *   CR0.PG) is 1.
3181	 */
3182	if (to_vmx(vcpu)->nested.nested_run_pending &&
3183	    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
3184		if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) ||
3185		    CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) ||
3186		    CC(((vmcs12->guest_cr0 & X86_CR0_PG) &&
3187		     ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))))
3188			return -EINVAL;
3189	}
3190
3191	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&
3192	    (CC(is_noncanonical_msr_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) ||
3193	     CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))))
3194		return -EINVAL;
3195
3196	if (nested_check_guest_non_reg_state(vmcs12))
3197		return -EINVAL;
3198
3199	return 0;
3200}
3201
3202static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu)
3203{
3204	struct vcpu_vmx *vmx = to_vmx(vcpu);
3205	unsigned long cr3, cr4;
3206	bool vm_fail;
3207
3208	if (!nested_early_check)
3209		return 0;
3210
3211	if (vmx->msr_autoload.host.nr)
3212		vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
3213	if (vmx->msr_autoload.guest.nr)
3214		vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
3215
3216	preempt_disable();
3217
3218	vmx_prepare_switch_to_guest(vcpu);
3219
3220	/*
3221	 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS,
3222	 * which is reserved to '1' by hardware.  GUEST_RFLAGS is guaranteed to
3223	 * be written (by prepare_vmcs02()) before the "real" VMEnter, i.e.
3224	 * there is no need to preserve other bits or save/restore the field.
3225	 */
3226	vmcs_writel(GUEST_RFLAGS, 0);
3227
3228	cr3 = __get_current_cr3_fast();
3229	if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
3230		vmcs_writel(HOST_CR3, cr3);
3231		vmx->loaded_vmcs->host_state.cr3 = cr3;
3232	}
3233
3234	cr4 = cr4_read_shadow();
3235	if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
3236		vmcs_writel(HOST_CR4, cr4);
3237		vmx->loaded_vmcs->host_state.cr4 = cr4;
3238	}
3239
3240	vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
3241				 __vmx_vcpu_run_flags(vmx));
3242
3243	if (vmx->msr_autoload.host.nr)
3244		vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
3245	if (vmx->msr_autoload.guest.nr)
3246		vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
3247
3248	if (vm_fail) {
3249		u32 error = vmcs_read32(VM_INSTRUCTION_ERROR);
3250
3251		preempt_enable();
3252
3253		trace_kvm_nested_vmenter_failed(
3254			"early hardware check VM-instruction error: ", error);
3255		WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3256		return 1;
3257	}
3258
3259	/*
3260	 * VMExit clears RFLAGS.IF and DR7, even on a consistency check.
3261	 */
3262	if (hw_breakpoint_active())
3263		set_debugreg(__this_cpu_read(cpu_dr7), 7);
3264	local_irq_enable();
3265	preempt_enable();
3266
3267	/*
3268	 * A non-failing VMEntry means we somehow entered guest mode with
3269	 * an illegal RIP, and that's just the tip of the iceberg.  There
3270	 * is no telling what memory has been modified or what state has
3271	 * been exposed to unknown code.  Hitting this all but guarantees
3272	 * a (very critical) hardware issue.
3273	 */
3274	WARN_ON(!(vmcs_read32(VM_EXIT_REASON) &
3275		VMX_EXIT_REASONS_FAILED_VMENTRY));
3276
3277	return 0;
3278}
3279
3280#ifdef CONFIG_KVM_HYPERV
3281static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu)
3282{
3283	struct vcpu_vmx *vmx = to_vmx(vcpu);
3284
3285	/*
3286	 * hv_evmcs may end up being not mapped after migration (when
3287	 * L2 was running), map it here to make sure vmcs12 changes are
3288	 * properly reflected.
3289	 */
3290	if (guest_cpuid_has_evmcs(vcpu) &&
3291	    vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) {
3292		enum nested_evmptrld_status evmptrld_status =
3293			nested_vmx_handle_enlightened_vmptrld(vcpu, false);
3294
3295		if (evmptrld_status == EVMPTRLD_VMFAIL ||
3296		    evmptrld_status == EVMPTRLD_ERROR)
3297			return false;
3298
3299		/*
3300		 * Post migration VMCS12 always provides the most actual
3301		 * information, copy it to eVMCS upon entry.
3302		 */
3303		vmx->nested.need_vmcs12_to_shadow_sync = true;
3304	}
3305
3306	return true;
3307}
3308#endif
3309
3310static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu)
3311{
3312	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3313	struct vcpu_vmx *vmx = to_vmx(vcpu);
3314	struct kvm_host_map *map;
3315
3316	if (!vcpu->arch.pdptrs_from_userspace &&
3317	    !nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) {
3318		/*
3319		 * Reload the guest's PDPTRs since after a migration
3320		 * the guest CR3 might be restored prior to setting the nested
3321		 * state which can lead to a load of wrong PDPTRs.
3322		 */
3323		if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3)))
3324			return false;
3325	}
3326
3327
3328	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
3329		map = &vmx->nested.apic_access_page_map;
3330
3331		if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->apic_access_addr), map)) {
3332			vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(map->pfn));
3333		} else {
3334			pr_debug_ratelimited("%s: no backing for APIC-access address in vmcs12\n",
3335					     __func__);
3336			vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3337			vcpu->run->internal.suberror =
3338				KVM_INTERNAL_ERROR_EMULATION;
3339			vcpu->run->internal.ndata = 0;
3340			return false;
3341		}
3342	}
3343
3344	if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
3345		map = &vmx->nested.virtual_apic_map;
3346
3347		if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) {
3348			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn));
3349		} else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) &&
3350		           nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) &&
3351			   !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
3352			/*
3353			 * The processor will never use the TPR shadow, simply
3354			 * clear the bit from the execution control.  Such a
3355			 * configuration is useless, but it happens in tests.
3356			 * For any other configuration, failing the vm entry is
3357			 * _not_ what the processor does but it's basically the
3358			 * only possibility we have.
3359			 */
3360			exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW);
3361		} else {
3362			/*
3363			 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to
3364			 * force VM-Entry to fail.
3365			 */
3366			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, INVALID_GPA);
3367		}
3368	}
3369
3370	if (nested_cpu_has_posted_intr(vmcs12)) {
3371		map = &vmx->nested.pi_desc_map;
3372
3373		if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) {
3374			vmx->nested.pi_desc =
3375				(struct pi_desc *)(((void *)map->hva) +
3376				offset_in_page(vmcs12->posted_intr_desc_addr));
3377			vmcs_write64(POSTED_INTR_DESC_ADDR,
3378				     pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr));
3379		} else {
3380			/*
3381			 * Defer the KVM_INTERNAL_EXIT until KVM tries to
3382			 * access the contents of the VMCS12 posted interrupt
3383			 * descriptor. (Note that KVM may do this when it
3384			 * should not, per the architectural specification.)
3385			 */
3386			vmx->nested.pi_desc = NULL;
3387			pin_controls_clearbit(vmx, PIN_BASED_POSTED_INTR);
3388		}
3389	}
3390	if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12))
3391		exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
3392	else
3393		exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
3394
3395	return true;
3396}
3397
3398static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu)
3399{
3400#ifdef CONFIG_KVM_HYPERV
3401	/*
3402	 * Note: nested_get_evmcs_page() also updates 'vp_assist_page' copy
3403	 * in 'struct kvm_vcpu_hv' in case eVMCS is in use, this is mandatory
3404	 * to make nested_evmcs_l2_tlb_flush_enabled() work correctly post
3405	 * migration.
3406	 */
3407	if (!nested_get_evmcs_page(vcpu)) {
3408		pr_debug_ratelimited("%s: enlightened vmptrld failed\n",
3409				     __func__);
3410		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3411		vcpu->run->internal.suberror =
3412			KVM_INTERNAL_ERROR_EMULATION;
3413		vcpu->run->internal.ndata = 0;
3414
3415		return false;
3416	}
3417#endif
3418
3419	if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu))
3420		return false;
3421
3422	return true;
3423}
3424
3425static int nested_vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa)
3426{
3427	struct vmcs12 *vmcs12;
3428	struct vcpu_vmx *vmx = to_vmx(vcpu);
3429	gpa_t dst;
3430
3431	if (WARN_ON_ONCE(!is_guest_mode(vcpu)))
3432		return 0;
3433
3434	if (WARN_ON_ONCE(vmx->nested.pml_full))
3435		return 1;
3436
3437	/*
3438	 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is
3439	 * set is already checked as part of A/D emulation.
3440	 */
3441	vmcs12 = get_vmcs12(vcpu);
3442	if (!nested_cpu_has_pml(vmcs12))
3443		return 0;
3444
3445	if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) {
3446		vmx->nested.pml_full = true;
3447		return 1;
3448	}
3449
3450	gpa &= ~0xFFFull;
3451	dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index;
3452
3453	if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa,
3454				 offset_in_page(dst), sizeof(gpa)))
3455		return 0;
3456
3457	vmcs12->guest_pml_index--;
3458
3459	return 0;
3460}
3461
3462/*
3463 * Intel's VMX Instruction Reference specifies a common set of prerequisites
3464 * for running VMX instructions (except VMXON, whose prerequisites are
3465 * slightly different). It also specifies what exception to inject otherwise.
3466 * Note that many of these exceptions have priority over VM exits, so they
3467 * don't have to be checked again here.
3468 */
3469static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
3470{
3471	if (!to_vmx(vcpu)->nested.vmxon) {
3472		kvm_queue_exception(vcpu, UD_VECTOR);
3473		return 0;
3474	}
3475
3476	if (vmx_get_cpl(vcpu)) {
3477		kvm_inject_gp(vcpu, 0);
3478		return 0;
3479	}
3480
3481	return 1;
3482}
3483
3484static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu)
3485{
3486	u8 rvi = vmx_get_rvi();
3487	u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI);
3488
3489	return ((rvi & 0xf0) > (vppr & 0xf0));
3490}
3491
3492static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
3493				   struct vmcs12 *vmcs12);
3494
3495/*
3496 * If from_vmentry is false, this is being called from state restore (either RSM
3497 * or KVM_SET_NESTED_STATE).  Otherwise it's called from vmlaunch/vmresume.
3498 *
3499 * Returns:
3500 *	NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode
3501 *	NVMX_VMENTRY_VMFAIL:  Consistency check VMFail
3502 *	NVMX_VMENTRY_VMEXIT:  Consistency check VMExit
3503 *	NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error
3504 */
3505enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
3506							bool from_vmentry)
3507{
3508	struct vcpu_vmx *vmx = to_vmx(vcpu);
3509	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3510	enum vm_entry_failure_code entry_failure_code;
3511	bool evaluate_pending_interrupts;
3512	union vmx_exit_reason exit_reason = {
3513		.basic = EXIT_REASON_INVALID_STATE,
3514		.failed_vmentry = 1,
3515	};
3516	u32 failed_index;
3517
3518	trace_kvm_nested_vmenter(kvm_rip_read(vcpu),
3519				 vmx->nested.current_vmptr,
3520				 vmcs12->guest_rip,
3521				 vmcs12->guest_intr_status,
3522				 vmcs12->vm_entry_intr_info_field,
3523				 vmcs12->secondary_vm_exec_control & SECONDARY_EXEC_ENABLE_EPT,
3524				 vmcs12->ept_pointer,
3525				 vmcs12->guest_cr3,
3526				 KVM_ISA_VMX);
3527
3528	kvm_service_local_tlb_flush_requests(vcpu);
3529
3530	evaluate_pending_interrupts = exec_controls_get(vmx) &
3531		(CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING);
3532	if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu))
3533		evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu);
3534	if (!evaluate_pending_interrupts)
3535		evaluate_pending_interrupts |= kvm_apic_has_pending_init_or_sipi(vcpu);
3536
3537	if (!vmx->nested.nested_run_pending ||
3538	    !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
3539		vmx->nested.pre_vmenter_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
3540	if (kvm_mpx_supported() &&
3541	    (!vmx->nested.nested_run_pending ||
3542	     !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)))
3543		vmx->nested.pre_vmenter_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
3544
3545	/*
3546	 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and*
3547	 * nested early checks are disabled.  In the event of a "late" VM-Fail,
3548	 * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its
3549	 * software model to the pre-VMEntry host state.  When EPT is disabled,
3550	 * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes
3551	 * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3.  Stuffing
3552	 * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to
3553	 * the correct value.  Smashing vmcs01.GUEST_CR3 is safe because nested
3554	 * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is
3555	 * guaranteed to be overwritten with a shadow CR3 prior to re-entering
3556	 * L1.  Don't stuff vmcs01.GUEST_CR3 when using nested early checks as
3557	 * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks
3558	 * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail
3559	 * path would need to manually save/restore vmcs01.GUEST_CR3.
3560	 */
3561	if (!enable_ept && !nested_early_check)
3562		vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
3563
3564	vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
3565
3566	prepare_vmcs02_early(vmx, &vmx->vmcs01, vmcs12);
3567
3568	if (from_vmentry) {
3569		if (unlikely(!nested_get_vmcs12_pages(vcpu))) {
3570			vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3571			return NVMX_VMENTRY_KVM_INTERNAL_ERROR;
3572		}
3573
3574		if (nested_vmx_check_vmentry_hw(vcpu)) {
3575			vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3576			return NVMX_VMENTRY_VMFAIL;
3577		}
3578
3579		if (nested_vmx_check_guest_state(vcpu, vmcs12,
3580						 &entry_failure_code)) {
3581			exit_reason.basic = EXIT_REASON_INVALID_STATE;
3582			vmcs12->exit_qualification = entry_failure_code;
3583			goto vmentry_fail_vmexit;
3584		}
3585	}
3586
3587	enter_guest_mode(vcpu);
3588
3589	if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &entry_failure_code)) {
3590		exit_reason.basic = EXIT_REASON_INVALID_STATE;
3591		vmcs12->exit_qualification = entry_failure_code;
3592		goto vmentry_fail_vmexit_guest_mode;
3593	}
3594
3595	if (from_vmentry) {
3596		failed_index = nested_vmx_load_msr(vcpu,
3597						   vmcs12->vm_entry_msr_load_addr,
3598						   vmcs12->vm_entry_msr_load_count);
3599		if (failed_index) {
3600			exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL;
3601			vmcs12->exit_qualification = failed_index;
3602			goto vmentry_fail_vmexit_guest_mode;
3603		}
3604	} else {
3605		/*
3606		 * The MMU is not initialized to point at the right entities yet and
3607		 * "get pages" would need to read data from the guest (i.e. we will
3608		 * need to perform gpa to hpa translation). Request a call
3609		 * to nested_get_vmcs12_pages before the next VM-entry.  The MSRs
3610		 * have already been set at vmentry time and should not be reset.
3611		 */
3612		kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
3613	}
3614
3615	/*
3616	 * Re-evaluate pending events if L1 had a pending IRQ/NMI/INIT/SIPI
3617	 * when it executed VMLAUNCH/VMRESUME, as entering non-root mode can
3618	 * effectively unblock various events, e.g. INIT/SIPI cause VM-Exit
3619	 * unconditionally.
3620	 */
3621	if (unlikely(evaluate_pending_interrupts))
3622		kvm_make_request(KVM_REQ_EVENT, vcpu);
3623
3624	/*
3625	 * Do not start the preemption timer hrtimer until after we know
3626	 * we are successful, so that only nested_vmx_vmexit needs to cancel
3627	 * the timer.
3628	 */
3629	vmx->nested.preemption_timer_expired = false;
3630	if (nested_cpu_has_preemption_timer(vmcs12)) {
3631		u64 timer_value = vmx_calc_preemption_timer_value(vcpu);
3632		vmx_start_preemption_timer(vcpu, timer_value);
3633	}
3634
3635	/*
3636	 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
3637	 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
3638	 * returned as far as L1 is concerned. It will only return (and set
3639	 * the success flag) when L2 exits (see nested_vmx_vmexit()).
3640	 */
3641	return NVMX_VMENTRY_SUCCESS;
3642
3643	/*
3644	 * A failed consistency check that leads to a VMExit during L1's
3645	 * VMEnter to L2 is a variation of a normal VMexit, as explained in
3646	 * 26.7 "VM-entry failures during or after loading guest state".
3647	 */
3648vmentry_fail_vmexit_guest_mode:
3649	if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
3650		vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
3651	leave_guest_mode(vcpu);
3652
3653vmentry_fail_vmexit:
3654	vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3655
3656	if (!from_vmentry)
3657		return NVMX_VMENTRY_VMEXIT;
3658
3659	load_vmcs12_host_state(vcpu, vmcs12);
3660	vmcs12->vm_exit_reason = exit_reason.full;
3661	if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx))
3662		vmx->nested.need_vmcs12_to_shadow_sync = true;
3663	return NVMX_VMENTRY_VMEXIT;
3664}
3665
3666/*
3667 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
3668 * for running an L2 nested guest.
3669 */
3670static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
3671{
3672	struct vmcs12 *vmcs12;
3673	enum nvmx_vmentry_status status;
3674	struct vcpu_vmx *vmx = to_vmx(vcpu);
3675	u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu);
3676	enum nested_evmptrld_status evmptrld_status;
3677
3678	if (!nested_vmx_check_permission(vcpu))
3679		return 1;
3680
3681	evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch);
3682	if (evmptrld_status == EVMPTRLD_ERROR) {
3683		kvm_queue_exception(vcpu, UD_VECTOR);
3684		return 1;
3685	}
3686
3687	kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED);
3688
3689	if (CC(evmptrld_status == EVMPTRLD_VMFAIL))
3690		return nested_vmx_failInvalid(vcpu);
3691
3692	if (CC(!nested_vmx_is_evmptr12_valid(vmx) &&
3693	       vmx->nested.current_vmptr == INVALID_GPA))
3694		return nested_vmx_failInvalid(vcpu);
3695
3696	vmcs12 = get_vmcs12(vcpu);
3697
3698	/*
3699	 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact
3700	 * that there *is* a valid VMCS pointer, RFLAGS.CF is set
3701	 * rather than RFLAGS.ZF, and no error number is stored to the
3702	 * VM-instruction error field.
3703	 */
3704	if (CC(vmcs12->hdr.shadow_vmcs))
3705		return nested_vmx_failInvalid(vcpu);
3706
3707	if (nested_vmx_is_evmptr12_valid(vmx)) {
3708		struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx);
3709
3710		copy_enlightened_to_vmcs12(vmx, evmcs->hv_clean_fields);
3711		/* Enlightened VMCS doesn't have launch state */
3712		vmcs12->launch_state = !launch;
3713	} else if (enable_shadow_vmcs) {
3714		copy_shadow_to_vmcs12(vmx);
3715	}
3716
3717	/*
3718	 * The nested entry process starts with enforcing various prerequisites
3719	 * on vmcs12 as required by the Intel SDM, and act appropriately when
3720	 * they fail: As the SDM explains, some conditions should cause the
3721	 * instruction to fail, while others will cause the instruction to seem
3722	 * to succeed, but return an EXIT_REASON_INVALID_STATE.
3723	 * To speed up the normal (success) code path, we should avoid checking
3724	 * for misconfigurations which will anyway be caught by the processor
3725	 * when using the merged vmcs02.
3726	 */
3727	if (CC(interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS))
3728		return nested_vmx_fail(vcpu, VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS);
3729
3730	if (CC(vmcs12->launch_state == launch))
3731		return nested_vmx_fail(vcpu,
3732			launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
3733			       : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
3734
3735	if (nested_vmx_check_controls(vcpu, vmcs12))
3736		return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3737
3738	if (nested_vmx_check_address_space_size(vcpu, vmcs12))
3739		return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
3740
3741	if (nested_vmx_check_host_state(vcpu, vmcs12))
3742		return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
3743
3744	/*
3745	 * We're finally done with prerequisite checking, and can start with
3746	 * the nested entry.
3747	 */
3748	vmx->nested.nested_run_pending = 1;
3749	vmx->nested.has_preemption_timer_deadline = false;
3750	status = nested_vmx_enter_non_root_mode(vcpu, true);
3751	if (unlikely(status != NVMX_VMENTRY_SUCCESS))
3752		goto vmentry_failed;
3753
3754	/* Emulate processing of posted interrupts on VM-Enter. */
3755	if (nested_cpu_has_posted_intr(vmcs12) &&
3756	    kvm_apic_has_interrupt(vcpu) == vmx->nested.posted_intr_nv) {
3757		vmx->nested.pi_pending = true;
3758		kvm_make_request(KVM_REQ_EVENT, vcpu);
3759		kvm_apic_clear_irr(vcpu, vmx->nested.posted_intr_nv);
3760	}
3761
3762	/* Hide L1D cache contents from the nested guest.  */
3763	vmx->vcpu.arch.l1tf_flush_l1d = true;
3764
3765	/*
3766	 * Must happen outside of nested_vmx_enter_non_root_mode() as it will
3767	 * also be used as part of restoring nVMX state for
3768	 * snapshot restore (migration).
3769	 *
3770	 * In this flow, it is assumed that vmcs12 cache was
3771	 * transferred as part of captured nVMX state and should
3772	 * therefore not be read from guest memory (which may not
3773	 * exist on destination host yet).
3774	 */
3775	nested_cache_shadow_vmcs12(vcpu, vmcs12);
3776
3777	switch (vmcs12->guest_activity_state) {
3778	case GUEST_ACTIVITY_HLT:
3779		/*
3780		 * If we're entering a halted L2 vcpu and the L2 vcpu won't be
3781		 * awakened by event injection or by an NMI-window VM-exit or
3782		 * by an interrupt-window VM-exit, halt the vcpu.
3783		 */
3784		if (!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) &&
3785		    !nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING) &&
3786		    !(nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING) &&
3787		      (vmcs12->guest_rflags & X86_EFLAGS_IF))) {
3788			vmx->nested.nested_run_pending = 0;
3789			return kvm_emulate_halt_noskip(vcpu);
3790		}
3791		break;
3792	case GUEST_ACTIVITY_WAIT_SIPI:
3793		vmx->nested.nested_run_pending = 0;
3794		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
3795		break;
3796	default:
3797		break;
3798	}
3799
3800	return 1;
3801
3802vmentry_failed:
3803	vmx->nested.nested_run_pending = 0;
3804	if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR)
3805		return 0;
3806	if (status == NVMX_VMENTRY_VMEXIT)
3807		return 1;
3808	WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL);
3809	return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3810}
3811
3812/*
3813 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
3814 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK).
3815 * This function returns the new value we should put in vmcs12.guest_cr0.
3816 * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
3817 *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
3818 *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
3819 *     didn't trap the bit, because if L1 did, so would L0).
3820 *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
3821 *     been modified by L2, and L1 knows it. So just leave the old value of
3822 *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
3823 *     isn't relevant, because if L0 traps this bit it can set it to anything.
3824 *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
3825 *     changed these bits, and therefore they need to be updated, but L0
3826 *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
3827 *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
3828 */
3829static inline unsigned long
3830vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3831{
3832	return
3833	/*1*/	(vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
3834	/*2*/	(vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
3835	/*3*/	(vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
3836			vcpu->arch.cr0_guest_owned_bits));
3837}
3838
3839static inline unsigned long
3840vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3841{
3842	return
3843	/*1*/	(vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
3844	/*2*/	(vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
3845	/*3*/	(vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
3846			vcpu->arch.cr4_guest_owned_bits));
3847}
3848
3849static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
3850				      struct vmcs12 *vmcs12,
3851				      u32 vm_exit_reason, u32 exit_intr_info)
3852{
3853	u32 idt_vectoring;
3854	unsigned int nr;
3855
3856	/*
3857	 * Per the SDM, VM-Exits due to double and triple faults are never
3858	 * considered to occur during event delivery, even if the double/triple
3859	 * fault is the result of an escalating vectoring issue.
3860	 *
3861	 * Note, the SDM qualifies the double fault behavior with "The original
3862	 * event results in a double-fault exception".  It's unclear why the
3863	 * qualification exists since exits due to double fault can occur only
3864	 * while vectoring a different exception (injected events are never
3865	 * subject to interception), i.e. there's _always_ an original event.
3866	 *
3867	 * The SDM also uses NMI as a confusing example for the "original event
3868	 * causes the VM exit directly" clause.  NMI isn't special in any way,
3869	 * the same rule applies to all events that cause an exit directly.
3870	 * NMI is an odd choice for the example because NMIs can only occur on
3871	 * instruction boundaries, i.e. they _can't_ occur during vectoring.
3872	 */
3873	if ((u16)vm_exit_reason == EXIT_REASON_TRIPLE_FAULT ||
3874	    ((u16)vm_exit_reason == EXIT_REASON_EXCEPTION_NMI &&
3875	     is_double_fault(exit_intr_info))) {
3876		vmcs12->idt_vectoring_info_field = 0;
3877	} else if (vcpu->arch.exception.injected) {
3878		nr = vcpu->arch.exception.vector;
3879		idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
3880
3881		if (kvm_exception_is_soft(nr)) {
3882			vmcs12->vm_exit_instruction_len =
3883				vcpu->arch.event_exit_inst_len;
3884			idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
3885		} else
3886			idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
3887
3888		if (vcpu->arch.exception.has_error_code) {
3889			idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
3890			vmcs12->idt_vectoring_error_code =
3891				vcpu->arch.exception.error_code;
3892		}
3893
3894		vmcs12->idt_vectoring_info_field = idt_vectoring;
3895	} else if (vcpu->arch.nmi_injected) {
3896		vmcs12->idt_vectoring_info_field =
3897			INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
3898	} else if (vcpu->arch.interrupt.injected) {
3899		nr = vcpu->arch.interrupt.nr;
3900		idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
3901
3902		if (vcpu->arch.interrupt.soft) {
3903			idt_vectoring |= INTR_TYPE_SOFT_INTR;
3904			vmcs12->vm_entry_instruction_len =
3905				vcpu->arch.event_exit_inst_len;
3906		} else
3907			idt_vectoring |= INTR_TYPE_EXT_INTR;
3908
3909		vmcs12->idt_vectoring_info_field = idt_vectoring;
3910	} else {
3911		vmcs12->idt_vectoring_info_field = 0;
3912	}
3913}
3914
3915
3916void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
3917{
3918	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3919	gfn_t gfn;
3920
3921	/*
3922	 * Don't need to mark the APIC access page dirty; it is never
3923	 * written to by the CPU during APIC virtualization.
3924	 */
3925
3926	if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
3927		gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
3928		kvm_vcpu_mark_page_dirty(vcpu, gfn);
3929	}
3930
3931	if (nested_cpu_has_posted_intr(vmcs12)) {
3932		gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
3933		kvm_vcpu_mark_page_dirty(vcpu, gfn);
3934	}
3935}
3936
3937static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
3938{
3939	struct vcpu_vmx *vmx = to_vmx(vcpu);
3940	int max_irr;
3941	void *vapic_page;
3942	u16 status;
3943
3944	if (!vmx->nested.pi_pending)
3945		return 0;
3946
3947	if (!vmx->nested.pi_desc)
3948		goto mmio_needed;
3949
3950	vmx->nested.pi_pending = false;
3951
3952	if (!pi_test_and_clear_on(vmx->nested.pi_desc))
3953		return 0;
3954
3955	max_irr = pi_find_highest_vector(vmx->nested.pi_desc);
3956	if (max_irr > 0) {
3957		vapic_page = vmx->nested.virtual_apic_map.hva;
3958		if (!vapic_page)
3959			goto mmio_needed;
3960
3961		__kvm_apic_update_irr(vmx->nested.pi_desc->pir,
3962			vapic_page, &max_irr);
3963		status = vmcs_read16(GUEST_INTR_STATUS);
3964		if ((u8)max_irr > ((u8)status & 0xff)) {
3965			status &= ~0xff;
3966			status |= (u8)max_irr;
3967			vmcs_write16(GUEST_INTR_STATUS, status);
3968		}
3969	}
3970
3971	nested_mark_vmcs12_pages_dirty(vcpu);
3972	return 0;
3973
3974mmio_needed:
3975	kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL);
3976	return -ENXIO;
3977}
3978
3979static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu)
3980{
3981	struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;
3982	u32 intr_info = ex->vector | INTR_INFO_VALID_MASK;
3983	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3984	unsigned long exit_qual;
3985
3986	if (ex->has_payload) {
3987		exit_qual = ex->payload;
3988	} else if (ex->vector == PF_VECTOR) {
3989		exit_qual = vcpu->arch.cr2;
3990	} else if (ex->vector == DB_VECTOR) {
3991		exit_qual = vcpu->arch.dr6;
3992		exit_qual &= ~DR6_BT;
3993		exit_qual ^= DR6_ACTIVE_LOW;
3994	} else {
3995		exit_qual = 0;
3996	}
3997
3998	/*
3999	 * Unlike AMD's Paged Real Mode, which reports an error code on #PF
4000	 * VM-Exits even if the CPU is in Real Mode, Intel VMX never sets the
4001	 * "has error code" flags on VM-Exit if the CPU is in Real Mode.
4002	 */
4003	if (ex->has_error_code && is_protmode(vcpu)) {
4004		/*
4005		 * Intel CPUs do not generate error codes with bits 31:16 set,
4006		 * and more importantly VMX disallows setting bits 31:16 in the
4007		 * injected error code for VM-Entry.  Drop the bits to mimic
4008		 * hardware and avoid inducing failure on nested VM-Entry if L1
4009		 * chooses to inject the exception back to L2.  AMD CPUs _do_
4010		 * generate "full" 32-bit error codes, so KVM allows userspace
4011		 * to inject exception error codes with bits 31:16 set.
4012		 */
4013		vmcs12->vm_exit_intr_error_code = (u16)ex->error_code;
4014		intr_info |= INTR_INFO_DELIVER_CODE_MASK;
4015	}
4016
4017	if (kvm_exception_is_soft(ex->vector))
4018		intr_info |= INTR_TYPE_SOFT_EXCEPTION;
4019	else
4020		intr_info |= INTR_TYPE_HARD_EXCEPTION;
4021
4022	if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
4023	    vmx_get_nmi_mask(vcpu))
4024		intr_info |= INTR_INFO_UNBLOCK_NMI;
4025
4026	nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
4027}
4028
4029/*
4030 * Returns true if a debug trap is (likely) pending delivery.  Infer the class
4031 * of a #DB (trap-like vs. fault-like) from the exception payload (to-be-DR6).
4032 * Using the payload is flawed because code breakpoints (fault-like) and data
4033 * breakpoints (trap-like) set the same bits in DR6 (breakpoint detected), i.e.
4034 * this will return false positives if a to-be-injected code breakpoint #DB is
4035 * pending (from KVM's perspective, but not "pending" across an instruction
4036 * boundary).  ICEBP, a.k.a. INT1, is also not reflected here even though it
4037 * too is trap-like.
4038 *
4039 * KVM "works" despite these flaws as ICEBP isn't currently supported by the
4040 * emulator, Monitor Trap Flag is not marked pending on intercepted #DBs (the
4041 * #DB has already happened), and MTF isn't marked pending on code breakpoints
4042 * from the emulator (because such #DBs are fault-like and thus don't trigger
4043 * actions that fire on instruction retire).
4044 */
4045static unsigned long vmx_get_pending_dbg_trap(struct kvm_queued_exception *ex)
4046{
4047	if (!ex->pending || ex->vector != DB_VECTOR)
4048		return 0;
4049
4050	/* General Detect #DBs are always fault-like. */
4051	return ex->payload & ~DR6_BD;
4052}
4053
4054/*
4055 * Returns true if there's a pending #DB exception that is lower priority than
4056 * a pending Monitor Trap Flag VM-Exit.  TSS T-flag #DBs are not emulated by
4057 * KVM, but could theoretically be injected by userspace.  Note, this code is
4058 * imperfect, see above.
4059 */
4060static bool vmx_is_low_priority_db_trap(struct kvm_queued_exception *ex)
4061{
4062	return vmx_get_pending_dbg_trap(ex) & ~DR6_BT;
4063}
4064
4065/*
4066 * Certain VM-exits set the 'pending debug exceptions' field to indicate a
4067 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM
4068 * represents these debug traps with a payload that is said to be compatible
4069 * with the 'pending debug exceptions' field, write the payload to the VMCS
4070 * field if a VM-exit is delivered before the debug trap.
4071 */
4072static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu)
4073{
4074	unsigned long pending_dbg;
4075
4076	pending_dbg = vmx_get_pending_dbg_trap(&vcpu->arch.exception);
4077	if (pending_dbg)
4078		vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, pending_dbg);
4079}
4080
4081static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu)
4082{
4083	return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
4084	       to_vmx(vcpu)->nested.preemption_timer_expired;
4085}
4086
4087static bool vmx_has_nested_events(struct kvm_vcpu *vcpu, bool for_injection)
4088{
4089	struct vcpu_vmx *vmx = to_vmx(vcpu);
4090	void *vapic = vmx->nested.virtual_apic_map.hva;
4091	int max_irr, vppr;
4092
4093	if (nested_vmx_preemption_timer_pending(vcpu) ||
4094	    vmx->nested.mtf_pending)
4095		return true;
4096
4097	/*
4098	 * Virtual Interrupt Delivery doesn't require manual injection.  Either
4099	 * the interrupt is already in GUEST_RVI and will be recognized by CPU
4100	 * at VM-Entry, or there is a KVM_REQ_EVENT pending and KVM will move
4101	 * the interrupt from the PIR to RVI prior to entering the guest.
4102	 */
4103	if (for_injection)
4104		return false;
4105
4106	if (!nested_cpu_has_vid(get_vmcs12(vcpu)) ||
4107	    __vmx_interrupt_blocked(vcpu))
4108		return false;
4109
4110	if (!vapic)
4111		return false;
4112
4113	vppr = *((u32 *)(vapic + APIC_PROCPRI));
4114
4115	max_irr = vmx_get_rvi();
4116	if ((max_irr & 0xf0) > (vppr & 0xf0))
4117		return true;
4118
4119	if (vmx->nested.pi_pending && vmx->nested.pi_desc &&
4120	    pi_test_on(vmx->nested.pi_desc)) {
4121		max_irr = pi_find_highest_vector(vmx->nested.pi_desc);
4122		if (max_irr > 0 && (max_irr & 0xf0) > (vppr & 0xf0))
4123			return true;
4124	}
4125
4126	return false;
4127}
4128
4129/*
4130 * Per the Intel SDM's table "Priority Among Concurrent Events", with minor
4131 * edits to fill in missing examples, e.g. #DB due to split-lock accesses,
4132 * and less minor edits to splice in the priority of VMX Non-Root specific
4133 * events, e.g. MTF and NMI/INTR-window exiting.
4134 *
4135 * 1 Hardware Reset and Machine Checks
4136 *	- RESET
4137 *	- Machine Check
4138 *
4139 * 2 Trap on Task Switch
4140 *	- T flag in TSS is set (on task switch)
4141 *
4142 * 3 External Hardware Interventions
4143 *	- FLUSH
4144 *	- STOPCLK
4145 *	- SMI
4146 *	- INIT
4147 *
4148 * 3.5 Monitor Trap Flag (MTF) VM-exit[1]
4149 *
4150 * 4 Traps on Previous Instruction
4151 *	- Breakpoints
4152 *	- Trap-class Debug Exceptions (#DB due to TF flag set, data/I-O
4153 *	  breakpoint, or #DB due to a split-lock access)
4154 *
4155 * 4.3	VMX-preemption timer expired VM-exit
4156 *
4157 * 4.6	NMI-window exiting VM-exit[2]
4158 *
4159 * 5 Nonmaskable Interrupts (NMI)
4160 *
4161 * 5.5 Interrupt-window exiting VM-exit and Virtual-interrupt delivery
4162 *
4163 * 6 Maskable Hardware Interrupts
4164 *
4165 * 7 Code Breakpoint Fault
4166 *
4167 * 8 Faults from Fetching Next Instruction
4168 *	- Code-Segment Limit Violation
4169 *	- Code Page Fault
4170 *	- Control protection exception (missing ENDBRANCH at target of indirect
4171 *					call or jump)
4172 *
4173 * 9 Faults from Decoding Next Instruction
4174 *	- Instruction length > 15 bytes
4175 *	- Invalid Opcode
4176 *	- Coprocessor Not Available
4177 *
4178 *10 Faults on Executing Instruction
4179 *	- Overflow
4180 *	- Bound error
4181 *	- Invalid TSS
4182 *	- Segment Not Present
4183 *	- Stack fault
4184 *	- General Protection
4185 *	- Data Page Fault
4186 *	- Alignment Check
4187 *	- x86 FPU Floating-point exception
4188 *	- SIMD floating-point exception
4189 *	- Virtualization exception
4190 *	- Control protection exception
4191 *
4192 * [1] Per the "Monitor Trap Flag" section: System-management interrupts (SMIs),
4193 *     INIT signals, and higher priority events take priority over MTF VM exits.
4194 *     MTF VM exits take priority over debug-trap exceptions and lower priority
4195 *     events.
4196 *
4197 * [2] Debug-trap exceptions and higher priority events take priority over VM exits
4198 *     caused by the VMX-preemption timer.  VM exits caused by the VMX-preemption
4199 *     timer take priority over VM exits caused by the "NMI-window exiting"
4200 *     VM-execution control and lower priority events.
4201 *
4202 * [3] Debug-trap exceptions and higher priority events take priority over VM exits
4203 *     caused by "NMI-window exiting".  VM exits caused by this control take
4204 *     priority over non-maskable interrupts (NMIs) and lower priority events.
4205 *
4206 * [4] Virtual-interrupt delivery has the same priority as that of VM exits due to
4207 *     the 1-setting of the "interrupt-window exiting" VM-execution control.  Thus,
4208 *     non-maskable interrupts (NMIs) and higher priority events take priority over
4209 *     delivery of a virtual interrupt; delivery of a virtual interrupt takes
4210 *     priority over external interrupts and lower priority events.
4211 */
4212static int vmx_check_nested_events(struct kvm_vcpu *vcpu)
4213{
4214	struct kvm_lapic *apic = vcpu->arch.apic;
4215	struct vcpu_vmx *vmx = to_vmx(vcpu);
4216	/*
4217	 * Only a pending nested run blocks a pending exception.  If there is a
4218	 * previously injected event, the pending exception occurred while said
4219	 * event was being delivered and thus needs to be handled.
4220	 */
4221	bool block_nested_exceptions = vmx->nested.nested_run_pending;
4222	/*
4223	 * New events (not exceptions) are only recognized at instruction
4224	 * boundaries.  If an event needs reinjection, then KVM is handling a
4225	 * VM-Exit that occurred _during_ instruction execution; new events are
4226	 * blocked until the instruction completes.
4227	 */
4228	bool block_nested_events = block_nested_exceptions ||
4229				   kvm_event_needs_reinjection(vcpu);
4230
4231	if (lapic_in_kernel(vcpu) &&
4232		test_bit(KVM_APIC_INIT, &apic->pending_events)) {
4233		if (block_nested_events)
4234			return -EBUSY;
4235		nested_vmx_update_pending_dbg(vcpu);
4236		clear_bit(KVM_APIC_INIT, &apic->pending_events);
4237		if (vcpu->arch.mp_state != KVM_MP_STATE_INIT_RECEIVED)
4238			nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0);
4239
4240		/* MTF is discarded if the vCPU is in WFS. */
4241		vmx->nested.mtf_pending = false;
4242		return 0;
4243	}
4244
4245	if (lapic_in_kernel(vcpu) &&
4246	    test_bit(KVM_APIC_SIPI, &apic->pending_events)) {
4247		if (block_nested_events)
4248			return -EBUSY;
4249
4250		clear_bit(KVM_APIC_SIPI, &apic->pending_events);
4251		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
4252			nested_vmx_vmexit(vcpu, EXIT_REASON_SIPI_SIGNAL, 0,
4253						apic->sipi_vector & 0xFFUL);
4254			return 0;
4255		}
4256		/* Fallthrough, the SIPI is completely ignored. */
4257	}
4258
4259	/*
4260	 * Process exceptions that are higher priority than Monitor Trap Flag:
4261	 * fault-like exceptions, TSS T flag #DB (not emulated by KVM, but
4262	 * could theoretically come in from userspace), and ICEBP (INT1).
4263	 *
4264	 * TODO: SMIs have higher priority than MTF and trap-like #DBs (except
4265	 * for TSS T flag #DBs).  KVM also doesn't save/restore pending MTF
4266	 * across SMI/RSM as it should; that needs to be addressed in order to
4267	 * prioritize SMI over MTF and trap-like #DBs.
4268	 */
4269	if (vcpu->arch.exception_vmexit.pending &&
4270	    !vmx_is_low_priority_db_trap(&vcpu->arch.exception_vmexit)) {
4271		if (block_nested_exceptions)
4272			return -EBUSY;
4273
4274		nested_vmx_inject_exception_vmexit(vcpu);
4275		return 0;
4276	}
4277
4278	if (vcpu->arch.exception.pending &&
4279	    !vmx_is_low_priority_db_trap(&vcpu->arch.exception)) {
4280		if (block_nested_exceptions)
4281			return -EBUSY;
4282		goto no_vmexit;
4283	}
4284
4285	if (vmx->nested.mtf_pending) {
4286		if (block_nested_events)
4287			return -EBUSY;
4288		nested_vmx_update_pending_dbg(vcpu);
4289		nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0);
4290		return 0;
4291	}
4292
4293	if (vcpu->arch.exception_vmexit.pending) {
4294		if (block_nested_exceptions)
4295			return -EBUSY;
4296
4297		nested_vmx_inject_exception_vmexit(vcpu);
4298		return 0;
4299	}
4300
4301	if (vcpu->arch.exception.pending) {
4302		if (block_nested_exceptions)
4303			return -EBUSY;
4304		goto no_vmexit;
4305	}
4306
4307	if (nested_vmx_preemption_timer_pending(vcpu)) {
4308		if (block_nested_events)
4309			return -EBUSY;
4310		nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
4311		return 0;
4312	}
4313
4314	if (vcpu->arch.smi_pending && !is_smm(vcpu)) {
4315		if (block_nested_events)
4316			return -EBUSY;
4317		goto no_vmexit;
4318	}
4319
4320	if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) {
4321		if (block_nested_events)
4322			return -EBUSY;
4323		if (!nested_exit_on_nmi(vcpu))
4324			goto no_vmexit;
4325
4326		nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
4327				  NMI_VECTOR | INTR_TYPE_NMI_INTR |
4328				  INTR_INFO_VALID_MASK, 0);
4329		/*
4330		 * The NMI-triggered VM exit counts as injection:
4331		 * clear this one and block further NMIs.
4332		 */
4333		vcpu->arch.nmi_pending = 0;
4334		vmx_set_nmi_mask(vcpu, true);
4335		return 0;
4336	}
4337
4338	if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) {
4339		int irq;
4340
4341		if (block_nested_events)
4342			return -EBUSY;
4343		if (!nested_exit_on_intr(vcpu))
4344			goto no_vmexit;
4345
4346		if (!nested_exit_intr_ack_set(vcpu)) {
4347			nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
4348			return 0;
4349		}
4350
4351		irq = kvm_cpu_get_extint(vcpu);
4352		if (irq != -1) {
4353			nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT,
4354					  INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR | irq, 0);
4355			return 0;
4356		}
4357
4358		irq = kvm_apic_has_interrupt(vcpu);
4359		if (WARN_ON_ONCE(irq < 0))
4360			goto no_vmexit;
4361
4362		/*
4363		 * If the IRQ is L2's PI notification vector, process posted
4364		 * interrupts for L2 instead of injecting VM-Exit, as the
4365		 * detection/morphing architecturally occurs when the IRQ is
4366		 * delivered to the CPU.  Note, only interrupts that are routed
4367		 * through the local APIC trigger posted interrupt processing,
4368		 * and enabling posted interrupts requires ACK-on-exit.
4369		 */
4370		if (irq == vmx->nested.posted_intr_nv) {
4371			vmx->nested.pi_pending = true;
4372			kvm_apic_clear_irr(vcpu, irq);
4373			goto no_vmexit;
4374		}
4375
4376		nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT,
4377				  INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR | irq, 0);
4378
4379		/*
4380		 * ACK the interrupt _after_ emulating VM-Exit, as the IRQ must
4381		 * be marked as in-service in vmcs01.GUEST_INTERRUPT_STATUS.SVI
4382		 * if APICv is active.
4383		 */
4384		kvm_apic_ack_interrupt(vcpu, irq);
4385		return 0;
4386	}
4387
4388no_vmexit:
4389	return vmx_complete_nested_posted_interrupt(vcpu);
4390}
4391
4392static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
4393{
4394	ktime_t remaining =
4395		hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
4396	u64 value;
4397
4398	if (ktime_to_ns(remaining) <= 0)
4399		return 0;
4400
4401	value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
4402	do_div(value, 1000000);
4403	return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
4404}
4405
4406static bool is_vmcs12_ext_field(unsigned long field)
4407{
4408	switch (field) {
4409	case GUEST_ES_SELECTOR:
4410	case GUEST_CS_SELECTOR:
4411	case GUEST_SS_SELECTOR:
4412	case GUEST_DS_SELECTOR:
4413	case GUEST_FS_SELECTOR:
4414	case GUEST_GS_SELECTOR:
4415	case GUEST_LDTR_SELECTOR:
4416	case GUEST_TR_SELECTOR:
4417	case GUEST_ES_LIMIT:
4418	case GUEST_CS_LIMIT:
4419	case GUEST_SS_LIMIT:
4420	case GUEST_DS_LIMIT:
4421	case GUEST_FS_LIMIT:
4422	case GUEST_GS_LIMIT:
4423	case GUEST_LDTR_LIMIT:
4424	case GUEST_TR_LIMIT:
4425	case GUEST_GDTR_LIMIT:
4426	case GUEST_IDTR_LIMIT:
4427	case GUEST_ES_AR_BYTES:
4428	case GUEST_DS_AR_BYTES:
4429	case GUEST_FS_AR_BYTES:
4430	case GUEST_GS_AR_BYTES:
4431	case GUEST_LDTR_AR_BYTES:
4432	case GUEST_TR_AR_BYTES:
4433	case GUEST_ES_BASE:
4434	case GUEST_CS_BASE:
4435	case GUEST_SS_BASE:
4436	case GUEST_DS_BASE:
4437	case GUEST_FS_BASE:
4438	case GUEST_GS_BASE:
4439	case GUEST_LDTR_BASE:
4440	case GUEST_TR_BASE:
4441	case GUEST_GDTR_BASE:
4442	case GUEST_IDTR_BASE:
4443	case GUEST_PENDING_DBG_EXCEPTIONS:
4444	case GUEST_BNDCFGS:
4445		return true;
4446	default:
4447		break;
4448	}
4449
4450	return false;
4451}
4452
4453static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
4454				       struct vmcs12 *vmcs12)
4455{
4456	struct vcpu_vmx *vmx = to_vmx(vcpu);
4457
4458	vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
4459	vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
4460	vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
4461	vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
4462	vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
4463	vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
4464	vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
4465	vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
4466	vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
4467	vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
4468	vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
4469	vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
4470	vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
4471	vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
4472	vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
4473	vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
4474	vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
4475	vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
4476	vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
4477	vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
4478	vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
4479	vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
4480	vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
4481	vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
4482	vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
4483	vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
4484	vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
4485	vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
4486	vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
4487	vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
4488	vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
4489	vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
4490	vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
4491	vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
4492	vmcs12->guest_pending_dbg_exceptions =
4493		vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
4494
4495	vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false;
4496}
4497
4498static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
4499				       struct vmcs12 *vmcs12)
4500{
4501	struct vcpu_vmx *vmx = to_vmx(vcpu);
4502	int cpu;
4503
4504	if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare)
4505		return;
4506
4507
4508	WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01);
4509
4510	cpu = get_cpu();
4511	vmx->loaded_vmcs = &vmx->nested.vmcs02;
4512	vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01);
4513
4514	sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
4515
4516	vmx->loaded_vmcs = &vmx->vmcs01;
4517	vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02);
4518	put_cpu();
4519}
4520
4521/*
4522 * Update the guest state fields of vmcs12 to reflect changes that
4523 * occurred while L2 was running. (The "IA-32e mode guest" bit of the
4524 * VM-entry controls is also updated, since this is really a guest
4525 * state bit.)
4526 */
4527static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
4528{
4529	struct vcpu_vmx *vmx = to_vmx(vcpu);
4530
4531	if (nested_vmx_is_evmptr12_valid(vmx))
4532		sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
4533
4534	vmx->nested.need_sync_vmcs02_to_vmcs12_rare =
4535		!nested_vmx_is_evmptr12_valid(vmx);
4536
4537	vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
4538	vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
4539
4540	vmcs12->guest_rsp = kvm_rsp_read(vcpu);
4541	vmcs12->guest_rip = kvm_rip_read(vcpu);
4542	vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
4543
4544	vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
4545	vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
4546
4547	vmcs12->guest_interruptibility_info =
4548		vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
4549
4550	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
4551		vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
4552	else if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
4553		vmcs12->guest_activity_state = GUEST_ACTIVITY_WAIT_SIPI;
4554	else
4555		vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
4556
4557	if (nested_cpu_has_preemption_timer(vmcs12) &&
4558	    vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER &&
4559	    !vmx->nested.nested_run_pending)
4560		vmcs12->vmx_preemption_timer_value =
4561			vmx_get_preemption_timer_value(vcpu);
4562
4563	/*
4564	 * In some cases (usually, nested EPT), L2 is allowed to change its
4565	 * own CR3 without exiting. If it has changed it, we must keep it.
4566	 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
4567	 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
4568	 *
4569	 * Additionally, restore L2's PDPTR to vmcs12.
4570	 */
4571	if (enable_ept) {
4572		vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
4573		if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) {
4574			vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
4575			vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
4576			vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
4577			vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
4578		}
4579	}
4580
4581	vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
4582
4583	if (nested_cpu_has_vid(vmcs12))
4584		vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
4585
4586	vmcs12->vm_entry_controls =
4587		(vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
4588		(vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
4589
4590	if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS)
4591		vmcs12->guest_dr7 = vcpu->arch.dr7;
4592
4593	if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
4594		vmcs12->guest_ia32_efer = vcpu->arch.efer;
4595}
4596
4597/*
4598 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
4599 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
4600 * and this function updates it to reflect the changes to the guest state while
4601 * L2 was running (and perhaps made some exits which were handled directly by L0
4602 * without going back to L1), and to reflect the exit reason.
4603 * Note that we do not have to copy here all VMCS fields, just those that
4604 * could have changed by the L2 guest or the exit - i.e., the guest-state and
4605 * exit-information fields only. Other fields are modified by L1 with VMWRITE,
4606 * which already writes to vmcs12 directly.
4607 */
4608static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
4609			   u32 vm_exit_reason, u32 exit_intr_info,
4610			   unsigned long exit_qualification)
4611{
4612	/* update exit information fields: */
4613	vmcs12->vm_exit_reason = vm_exit_reason;
4614	if (to_vmx(vcpu)->exit_reason.enclave_mode)
4615		vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE;
4616	vmcs12->exit_qualification = exit_qualification;
4617
4618	/*
4619	 * On VM-Exit due to a failed VM-Entry, the VMCS isn't marked launched
4620	 * and only EXIT_REASON and EXIT_QUALIFICATION are updated, all other
4621	 * exit info fields are unmodified.
4622	 */
4623	if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
4624		vmcs12->launch_state = 1;
4625
4626		/* vm_entry_intr_info_field is cleared on exit. Emulate this
4627		 * instead of reading the real value. */
4628		vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
4629
4630		/*
4631		 * Transfer the event that L0 or L1 may wanted to inject into
4632		 * L2 to IDT_VECTORING_INFO_FIELD.
4633		 */
4634		vmcs12_save_pending_event(vcpu, vmcs12,
4635					  vm_exit_reason, exit_intr_info);
4636
4637		vmcs12->vm_exit_intr_info = exit_intr_info;
4638		vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
4639		vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4640
4641		/*
4642		 * According to spec, there's no need to store the guest's
4643		 * MSRs if the exit is due to a VM-entry failure that occurs
4644		 * during or after loading the guest state. Since this exit
4645		 * does not fall in that category, we need to save the MSRs.
4646		 */
4647		if (nested_vmx_store_msr(vcpu,
4648					 vmcs12->vm_exit_msr_store_addr,
4649					 vmcs12->vm_exit_msr_store_count))
4650			nested_vmx_abort(vcpu,
4651					 VMX_ABORT_SAVE_GUEST_MSR_FAIL);
4652	}
4653}
4654
4655/*
4656 * A part of what we need to when the nested L2 guest exits and we want to
4657 * run its L1 parent, is to reset L1's guest state to the host state specified
4658 * in vmcs12.
4659 * This function is to be called not only on normal nested exit, but also on
4660 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
4661 * Failures During or After Loading Guest State").
4662 * This function should be called when the active VMCS is L1's (vmcs01).
4663 */
4664static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
4665				   struct vmcs12 *vmcs12)
4666{
4667	enum vm_entry_failure_code ignored;
4668	struct kvm_segment seg;
4669
4670	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
4671		vcpu->arch.efer = vmcs12->host_ia32_efer;
4672	else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
4673		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
4674	else
4675		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
4676	vmx_set_efer(vcpu, vcpu->arch.efer);
4677
4678	kvm_rsp_write(vcpu, vmcs12->host_rsp);
4679	kvm_rip_write(vcpu, vmcs12->host_rip);
4680	vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
4681	vmx_set_interrupt_shadow(vcpu, 0);
4682
4683	/*
4684	 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
4685	 * actually changed, because vmx_set_cr0 refers to efer set above.
4686	 *
4687	 * CR0_GUEST_HOST_MASK is already set in the original vmcs01
4688	 * (KVM doesn't change it);
4689	 */
4690	vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits();
4691	vmx_set_cr0(vcpu, vmcs12->host_cr0);
4692
4693	/* Same as above - no reason to call set_cr4_guest_host_mask().  */
4694	vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
4695	vmx_set_cr4(vcpu, vmcs12->host_cr4);
4696
4697	nested_ept_uninit_mmu_context(vcpu);
4698
4699	/*
4700	 * Only PDPTE load can fail as the value of cr3 was checked on entry and
4701	 * couldn't have changed.
4702	 */
4703	if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, true, &ignored))
4704		nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
4705
4706	nested_vmx_transition_tlb_flush(vcpu, vmcs12, false);
4707
4708	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
4709	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
4710	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
4711	vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
4712	vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
4713	vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF);
4714	vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);
4715
4716	/* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1.  */
4717	if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
4718		vmcs_write64(GUEST_BNDCFGS, 0);
4719
4720	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
4721		vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
4722		vcpu->arch.pat = vmcs12->host_ia32_pat;
4723	}
4724	if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) &&
4725	    kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)))
4726		WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL,
4727					 vmcs12->host_ia32_perf_global_ctrl));
4728
4729	/* Set L1 segment info according to Intel SDM
4730	    27.5.2 Loading Host Segment and Descriptor-Table Registers */
4731	seg = (struct kvm_segment) {
4732		.base = 0,
4733		.limit = 0xFFFFFFFF,
4734		.selector = vmcs12->host_cs_selector,
4735		.type = 11,
4736		.present = 1,
4737		.s = 1,
4738		.g = 1
4739	};
4740	if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
4741		seg.l = 1;
4742	else
4743		seg.db = 1;
4744	__vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
4745	seg = (struct kvm_segment) {
4746		.base = 0,
4747		.limit = 0xFFFFFFFF,
4748		.type = 3,
4749		.present = 1,
4750		.s = 1,
4751		.db = 1,
4752		.g = 1
4753	};
4754	seg.selector = vmcs12->host_ds_selector;
4755	__vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
4756	seg.selector = vmcs12->host_es_selector;
4757	__vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
4758	seg.selector = vmcs12->host_ss_selector;
4759	__vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
4760	seg.selector = vmcs12->host_fs_selector;
4761	seg.base = vmcs12->host_fs_base;
4762	__vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
4763	seg.selector = vmcs12->host_gs_selector;
4764	seg.base = vmcs12->host_gs_base;
4765	__vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
4766	seg = (struct kvm_segment) {
4767		.base = vmcs12->host_tr_base,
4768		.limit = 0x67,
4769		.selector = vmcs12->host_tr_selector,
4770		.type = 11,
4771		.present = 1
4772	};
4773	__vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
4774
4775	memset(&seg, 0, sizeof(seg));
4776	seg.unusable = 1;
4777	__vmx_set_segment(vcpu, &seg, VCPU_SREG_LDTR);
4778
4779	kvm_set_dr(vcpu, 7, 0x400);
4780	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
4781
4782	if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
4783				vmcs12->vm_exit_msr_load_count))
4784		nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
4785
4786	to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu);
4787}
4788
4789static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx)
4790{
4791	struct vmx_uret_msr *efer_msr;
4792	unsigned int i;
4793
4794	if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER)
4795		return vmcs_read64(GUEST_IA32_EFER);
4796
4797	if (cpu_has_load_ia32_efer())
4798		return kvm_host.efer;
4799
4800	for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) {
4801		if (vmx->msr_autoload.guest.val[i].index == MSR_EFER)
4802			return vmx->msr_autoload.guest.val[i].value;
4803	}
4804
4805	efer_msr = vmx_find_uret_msr(vmx, MSR_EFER);
4806	if (efer_msr)
4807		return efer_msr->data;
4808
4809	return kvm_host.efer;
4810}
4811
4812static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
4813{
4814	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4815	struct vcpu_vmx *vmx = to_vmx(vcpu);
4816	struct vmx_msr_entry g, h;
4817	gpa_t gpa;
4818	u32 i, j;
4819
4820	vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT);
4821
4822	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
4823		/*
4824		 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set
4825		 * as vmcs01.GUEST_DR7 contains a userspace defined value
4826		 * and vcpu->arch.dr7 is not squirreled away before the
4827		 * nested VMENTER (not worth adding a variable in nested_vmx).
4828		 */
4829		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
4830			kvm_set_dr(vcpu, 7, DR7_FIXED_1);
4831		else
4832			WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7)));
4833	}
4834
4835	/*
4836	 * Note that calling vmx_set_{efer,cr0,cr4} is important as they
4837	 * handle a variety of side effects to KVM's software model.
4838	 */
4839	vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx));
4840
4841	vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits();
4842	vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW));
4843
4844	vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
4845	vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW));
4846
4847	nested_ept_uninit_mmu_context(vcpu);
4848	vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
4849	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
4850
4851	/*
4852	 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs
4853	 * from vmcs01 (if necessary).  The PDPTRs are not loaded on
4854	 * VMFail, like everything else we just need to ensure our
4855	 * software model is up-to-date.
4856	 */
4857	if (enable_ept && is_pae_paging(vcpu))
4858		ept_save_pdptrs(vcpu);
4859
4860	kvm_mmu_reset_context(vcpu);
4861
4862	/*
4863	 * This nasty bit of open coding is a compromise between blindly
4864	 * loading L1's MSRs using the exit load lists (incorrect emulation
4865	 * of VMFail), leaving the nested VM's MSRs in the software model
4866	 * (incorrect behavior) and snapshotting the modified MSRs (too
4867	 * expensive since the lists are unbound by hardware).  For each
4868	 * MSR that was (prematurely) loaded from the nested VMEntry load
4869	 * list, reload it from the exit load list if it exists and differs
4870	 * from the guest value.  The intent is to stuff host state as
4871	 * silently as possible, not to fully process the exit load list.
4872	 */
4873	for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {
4874		gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));
4875		if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {
4876			pr_debug_ratelimited(
4877				"%s read MSR index failed (%u, 0x%08llx)\n",
4878				__func__, i, gpa);
4879			goto vmabort;
4880		}
4881
4882		for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) {
4883			gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h));
4884			if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) {
4885				pr_debug_ratelimited(
4886					"%s read MSR failed (%u, 0x%08llx)\n",
4887					__func__, j, gpa);
4888				goto vmabort;
4889			}
4890			if (h.index != g.index)
4891				continue;
4892			if (h.value == g.value)
4893				break;
4894
4895			if (nested_vmx_load_msr_check(vcpu, &h)) {
4896				pr_debug_ratelimited(
4897					"%s check failed (%u, 0x%x, 0x%x)\n",
4898					__func__, j, h.index, h.reserved);
4899				goto vmabort;
4900			}
4901
4902			if (kvm_set_msr_with_filter(vcpu, h.index, h.value)) {
4903				pr_debug_ratelimited(
4904					"%s WRMSR failed (%u, 0x%x, 0x%llx)\n",
4905					__func__, j, h.index, h.value);
4906				goto vmabort;
4907			}
4908		}
4909	}
4910
4911	return;
4912
4913vmabort:
4914	nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
4915}
4916
4917/*
4918 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
4919 * and modify vmcs12 to make it see what it would expect to see there if
4920 * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
4921 */
4922void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
4923		       u32 exit_intr_info, unsigned long exit_qualification)
4924{
4925	struct vcpu_vmx *vmx = to_vmx(vcpu);
4926	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4927
4928	/* Pending MTF traps are discarded on VM-Exit. */
4929	vmx->nested.mtf_pending = false;
4930
4931	/* trying to cancel vmlaunch/vmresume is a bug */
4932	WARN_ON_ONCE(vmx->nested.nested_run_pending);
4933
4934#ifdef CONFIG_KVM_HYPERV
4935	if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
4936		/*
4937		 * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map
4938		 * Enlightened VMCS after migration and we still need to
4939		 * do that when something is forcing L2->L1 exit prior to
4940		 * the first L2 run.
4941		 */
4942		(void)nested_get_evmcs_page(vcpu);
4943	}
4944#endif
4945
4946	/* Service pending TLB flush requests for L2 before switching to L1. */
4947	kvm_service_local_tlb_flush_requests(vcpu);
4948
4949	/*
4950	 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between
4951	 * now and the new vmentry.  Ensure that the VMCS02 PDPTR fields are
4952	 * up-to-date before switching to L1.
4953	 */
4954	if (enable_ept && is_pae_paging(vcpu))
4955		vmx_ept_load_pdptrs(vcpu);
4956
4957	leave_guest_mode(vcpu);
4958
4959	if (nested_cpu_has_preemption_timer(vmcs12))
4960		hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
4961
4962	if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) {
4963		vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset;
4964		if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING))
4965			vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
4966	}
4967
4968	if (likely(!vmx->fail)) {
4969		sync_vmcs02_to_vmcs12(vcpu, vmcs12);
4970
4971		if (vm_exit_reason != -1)
4972			prepare_vmcs12(vcpu, vmcs12, vm_exit_reason,
4973				       exit_intr_info, exit_qualification);
4974
4975		/*
4976		 * Must happen outside of sync_vmcs02_to_vmcs12() as it will
4977		 * also be used to capture vmcs12 cache as part of
4978		 * capturing nVMX state for snapshot (migration).
4979		 *
4980		 * Otherwise, this flush will dirty guest memory at a
4981		 * point it is already assumed by user-space to be
4982		 * immutable.
4983		 */
4984		nested_flush_cached_shadow_vmcs12(vcpu, vmcs12);
4985	} else {
4986		/*
4987		 * The only expected VM-instruction error is "VM entry with
4988		 * invalid control field(s)." Anything else indicates a
4989		 * problem with L0.  And we should never get here with a
4990		 * VMFail of any type if early consistency checks are enabled.
4991		 */
4992		WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=
4993			     VMXERR_ENTRY_INVALID_CONTROL_FIELD);
4994		WARN_ON_ONCE(nested_early_check);
4995	}
4996
4997	/*
4998	 * Drop events/exceptions that were queued for re-injection to L2
4999	 * (picked up via vmx_complete_interrupts()), as well as exceptions
5000	 * that were pending for L2.  Note, this must NOT be hoisted above
5001	 * prepare_vmcs12(), events/exceptions queued for re-injection need to
5002	 * be captured in vmcs12 (see vmcs12_save_pending_event()).
5003	 */
5004	vcpu->arch.nmi_injected = false;
5005	kvm_clear_exception_queue(vcpu);
5006	kvm_clear_interrupt_queue(vcpu);
5007
5008	vmx_switch_vmcs(vcpu, &vmx->vmcs01);
5009
5010	/*
5011	 * If IBRS is advertised to the vCPU, KVM must flush the indirect
5012	 * branch predictors when transitioning from L2 to L1, as L1 expects
5013	 * hardware (KVM in this case) to provide separate predictor modes.
5014	 * Bare metal isolates VMX root (host) from VMX non-root (guest), but
5015	 * doesn't isolate different VMCSs, i.e. in this case, doesn't provide
5016	 * separate modes for L2 vs L1.
5017	 */
5018	if (guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
5019		indirect_branch_prediction_barrier();
5020
5021	/* Update any VMCS fields that might have changed while L2 ran */
5022	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
5023	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
5024	vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
5025	if (kvm_caps.has_tsc_control)
5026		vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio);
5027
5028	if (vmx->nested.l1_tpr_threshold != -1)
5029		vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold);
5030
5031	if (vmx->nested.change_vmcs01_virtual_apic_mode) {
5032		vmx->nested.change_vmcs01_virtual_apic_mode = false;
5033		vmx_set_virtual_apic_mode(vcpu);
5034	}
5035
5036	if (vmx->nested.update_vmcs01_cpu_dirty_logging) {
5037		vmx->nested.update_vmcs01_cpu_dirty_logging = false;
5038		vmx_update_cpu_dirty_logging(vcpu);
5039	}
5040
5041	nested_put_vmcs12_pages(vcpu);
 
 
 
 
5042
5043	if (vmx->nested.reload_vmcs01_apic_access_page) {
5044		vmx->nested.reload_vmcs01_apic_access_page = false;
5045		kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
5046	}
5047
5048	if (vmx->nested.update_vmcs01_apicv_status) {
5049		vmx->nested.update_vmcs01_apicv_status = false;
5050		kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
5051	}
5052
5053	if (vmx->nested.update_vmcs01_hwapic_isr) {
5054		vmx->nested.update_vmcs01_hwapic_isr = false;
5055		kvm_apic_update_hwapic_isr(vcpu);
5056	}
5057
5058	if ((vm_exit_reason != -1) &&
5059	    (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)))
5060		vmx->nested.need_vmcs12_to_shadow_sync = true;
5061
5062	/* in case we halted in L2 */
5063	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
5064
5065	if (likely(!vmx->fail)) {
 
 
 
 
 
 
 
 
5066		if (vm_exit_reason != -1)
5067			trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
5068						       vmcs12->exit_qualification,
5069						       vmcs12->idt_vectoring_info_field,
5070						       vmcs12->vm_exit_intr_info,
5071						       vmcs12->vm_exit_intr_error_code,
5072						       KVM_ISA_VMX);
5073
5074		load_vmcs12_host_state(vcpu, vmcs12);
5075
5076		return;
5077	}
5078
5079	/*
5080	 * After an early L2 VM-entry failure, we're now back
5081	 * in L1 which thinks it just finished a VMLAUNCH or
5082	 * VMRESUME instruction, so we need to set the failure
5083	 * flag and the VM-instruction error field of the VMCS
5084	 * accordingly, and skip the emulated instruction.
5085	 */
5086	(void)nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
5087
5088	/*
5089	 * Restore L1's host state to KVM's software model.  We're here
5090	 * because a consistency check was caught by hardware, which
5091	 * means some amount of guest state has been propagated to KVM's
5092	 * model and needs to be unwound to the host's state.
5093	 */
5094	nested_vmx_restore_host_state(vcpu);
5095
5096	vmx->fail = 0;
5097}
5098
5099static void nested_vmx_triple_fault(struct kvm_vcpu *vcpu)
5100{
5101	kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5102	nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0);
5103}
5104
5105/*
5106 * Decode the memory-address operand of a vmx instruction, as recorded on an
5107 * exit caused by such an instruction (run by a guest hypervisor).
5108 * On success, returns 0. When the operand is invalid, returns 1 and throws
5109 * #UD, #GP, or #SS.
5110 */
5111int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification,
5112			u32 vmx_instruction_info, bool wr, int len, gva_t *ret)
5113{
5114	gva_t off;
5115	bool exn;
5116	struct kvm_segment s;
5117
5118	/*
5119	 * According to Vol. 3B, "Information for VM Exits Due to Instruction
5120	 * Execution", on an exit, vmx_instruction_info holds most of the
5121	 * addressing components of the operand. Only the displacement part
5122	 * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
5123	 * For how an actual address is calculated from all these components,
5124	 * refer to Vol. 1, "Operand Addressing".
5125	 */
5126	int  scaling = vmx_instruction_info & 3;
5127	int  addr_size = (vmx_instruction_info >> 7) & 7;
5128	bool is_reg = vmx_instruction_info & (1u << 10);
5129	int  seg_reg = (vmx_instruction_info >> 15) & 7;
5130	int  index_reg = (vmx_instruction_info >> 18) & 0xf;
5131	bool index_is_valid = !(vmx_instruction_info & (1u << 22));
5132	int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
5133	bool base_is_valid  = !(vmx_instruction_info & (1u << 27));
5134
5135	if (is_reg) {
5136		kvm_queue_exception(vcpu, UD_VECTOR);
5137		return 1;
5138	}
5139
5140	/* Addr = segment_base + offset */
5141	/* offset = base + [index * scale] + displacement */
5142	off = exit_qualification; /* holds the displacement */
5143	if (addr_size == 1)
5144		off = (gva_t)sign_extend64(off, 31);
5145	else if (addr_size == 0)
5146		off = (gva_t)sign_extend64(off, 15);
5147	if (base_is_valid)
5148		off += kvm_register_read(vcpu, base_reg);
5149	if (index_is_valid)
5150		off += kvm_register_read(vcpu, index_reg) << scaling;
5151	vmx_get_segment(vcpu, &s, seg_reg);
5152
5153	/*
5154	 * The effective address, i.e. @off, of a memory operand is truncated
5155	 * based on the address size of the instruction.  Note that this is
5156	 * the *effective address*, i.e. the address prior to accounting for
5157	 * the segment's base.
5158	 */
5159	if (addr_size == 1) /* 32 bit */
5160		off &= 0xffffffff;
5161	else if (addr_size == 0) /* 16 bit */
5162		off &= 0xffff;
5163
5164	/* Checks for #GP/#SS exceptions. */
5165	exn = false;
5166	if (is_long_mode(vcpu)) {
5167		/*
5168		 * The virtual/linear address is never truncated in 64-bit
5169		 * mode, e.g. a 32-bit address size can yield a 64-bit virtual
5170		 * address when using FS/GS with a non-zero base.
5171		 */
5172		if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS)
5173			*ret = s.base + off;
5174		else
5175			*ret = off;
5176
5177		*ret = vmx_get_untagged_addr(vcpu, *ret, 0);
5178		/* Long mode: #GP(0)/#SS(0) if the memory address is in a
5179		 * non-canonical form. This is the only check on the memory
5180		 * destination for long mode!
5181		 */
5182		exn = is_noncanonical_address(*ret, vcpu, 0);
5183	} else {
5184		/*
5185		 * When not in long mode, the virtual/linear address is
5186		 * unconditionally truncated to 32 bits regardless of the
5187		 * address size.
5188		 */
5189		*ret = (s.base + off) & 0xffffffff;
5190
5191		/* Protected mode: apply checks for segment validity in the
5192		 * following order:
5193		 * - segment type check (#GP(0) may be thrown)
5194		 * - usability check (#GP(0)/#SS(0))
5195		 * - limit check (#GP(0)/#SS(0))
5196		 */
5197		if (wr)
5198			/* #GP(0) if the destination operand is located in a
5199			 * read-only data segment or any code segment.
5200			 */
5201			exn = ((s.type & 0xa) == 0 || (s.type & 8));
5202		else
5203			/* #GP(0) if the source operand is located in an
5204			 * execute-only code segment
5205			 */
5206			exn = ((s.type & 0xa) == 8);
5207		if (exn) {
5208			kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
5209			return 1;
5210		}
5211		/* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
5212		 */
5213		exn = (s.unusable != 0);
5214
5215		/*
5216		 * Protected mode: #GP(0)/#SS(0) if the memory operand is
5217		 * outside the segment limit.  All CPUs that support VMX ignore
5218		 * limit checks for flat segments, i.e. segments with base==0,
5219		 * limit==0xffffffff and of type expand-up data or code.
5220		 */
5221		if (!(s.base == 0 && s.limit == 0xffffffff &&
5222		     ((s.type & 8) || !(s.type & 4))))
5223			exn = exn || ((u64)off + len - 1 > s.limit);
5224	}
5225	if (exn) {
5226		kvm_queue_exception_e(vcpu,
5227				      seg_reg == VCPU_SREG_SS ?
5228						SS_VECTOR : GP_VECTOR,
5229				      0);
5230		return 1;
5231	}
5232
5233	return 0;
5234}
5235
5236static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer,
5237				int *ret)
5238{
5239	gva_t gva;
5240	struct x86_exception e;
5241	int r;
5242
5243	if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
5244				vmcs_read32(VMX_INSTRUCTION_INFO), false,
5245				sizeof(*vmpointer), &gva)) {
5246		*ret = 1;
5247		return -EINVAL;
5248	}
5249
5250	r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e);
5251	if (r != X86EMUL_CONTINUE) {
5252		*ret = kvm_handle_memory_failure(vcpu, r, &e);
5253		return -EINVAL;
5254	}
5255
5256	return 0;
5257}
5258
5259/*
5260 * Allocate a shadow VMCS and associate it with the currently loaded
5261 * VMCS, unless such a shadow VMCS already exists. The newly allocated
5262 * VMCS is also VMCLEARed, so that it is ready for use.
5263 */
5264static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu)
5265{
5266	struct vcpu_vmx *vmx = to_vmx(vcpu);
5267	struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs;
5268
5269	/*
5270	 * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it
5271	 * when L1 executes VMXOFF or the vCPU is forced out of nested
5272	 * operation.  VMXON faults if the CPU is already post-VMXON, so it
5273	 * should be impossible to already have an allocated shadow VMCS.  KVM
5274	 * doesn't support virtualization of VMCS shadowing, so vmcs01 should
5275	 * always be the loaded VMCS.
5276	 */
5277	if (WARN_ON(loaded_vmcs != &vmx->vmcs01 || loaded_vmcs->shadow_vmcs))
5278		return loaded_vmcs->shadow_vmcs;
5279
5280	loaded_vmcs->shadow_vmcs = alloc_vmcs(true);
5281	if (loaded_vmcs->shadow_vmcs)
5282		vmcs_clear(loaded_vmcs->shadow_vmcs);
5283
5284	return loaded_vmcs->shadow_vmcs;
5285}
5286
5287static int enter_vmx_operation(struct kvm_vcpu *vcpu)
5288{
5289	struct vcpu_vmx *vmx = to_vmx(vcpu);
5290	int r;
5291
5292	r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
5293	if (r < 0)
5294		goto out_vmcs02;
5295
5296	vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
5297	if (!vmx->nested.cached_vmcs12)
5298		goto out_cached_vmcs12;
5299
5300	vmx->nested.shadow_vmcs12_cache.gpa = INVALID_GPA;
5301	vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
5302	if (!vmx->nested.cached_shadow_vmcs12)
5303		goto out_cached_shadow_vmcs12;
5304
5305	if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu))
5306		goto out_shadow_vmcs;
5307
5308	hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
5309		     HRTIMER_MODE_ABS_PINNED);
5310	vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
5311
5312	vmx->nested.vpid02 = allocate_vpid();
5313
5314	vmx->nested.vmcs02_initialized = false;
5315	vmx->nested.vmxon = true;
5316
5317	if (vmx_pt_mode_is_host_guest()) {
5318		vmx->pt_desc.guest.ctl = 0;
5319		pt_update_intercept_for_msr(vcpu);
5320	}
5321
5322	return 0;
5323
5324out_shadow_vmcs:
5325	kfree(vmx->nested.cached_shadow_vmcs12);
5326
5327out_cached_shadow_vmcs12:
5328	kfree(vmx->nested.cached_vmcs12);
5329
5330out_cached_vmcs12:
5331	free_loaded_vmcs(&vmx->nested.vmcs02);
5332
5333out_vmcs02:
5334	return -ENOMEM;
5335}
5336
5337/* Emulate the VMXON instruction. */
5338static int handle_vmxon(struct kvm_vcpu *vcpu)
5339{
5340	int ret;
5341	gpa_t vmptr;
5342	uint32_t revision;
5343	struct vcpu_vmx *vmx = to_vmx(vcpu);
5344	const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED
5345		| FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX;
5346
5347	/*
5348	 * Manually check CR4.VMXE checks, KVM must force CR4.VMXE=1 to enter
5349	 * the guest and so cannot rely on hardware to perform the check,
5350	 * which has higher priority than VM-Exit (see Intel SDM's pseudocode
5351	 * for VMXON).
5352	 *
5353	 * Rely on hardware for the other pre-VM-Exit checks, CR0.PE=1, !VM86
5354	 * and !COMPATIBILITY modes.  For an unrestricted guest, KVM doesn't
5355	 * force any of the relevant guest state.  For a restricted guest, KVM
5356	 * does force CR0.PE=1, but only to also force VM86 in order to emulate
5357	 * Real Mode, and so there's no need to check CR0.PE manually.
5358	 */
5359	if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_VMXE)) {
5360		kvm_queue_exception(vcpu, UD_VECTOR);
5361		return 1;
5362	}
5363
5364	/*
5365	 * The CPL is checked for "not in VMX operation" and for "in VMX root",
5366	 * and has higher priority than the VM-Fail due to being post-VMXON,
5367	 * i.e. VMXON #GPs outside of VMX non-root if CPL!=0.  In VMX non-root,
5368	 * VMXON causes VM-Exit and KVM unconditionally forwards VMXON VM-Exits
5369	 * from L2 to L1, i.e. there's no need to check for the vCPU being in
5370	 * VMX non-root.
5371	 *
5372	 * Forwarding the VM-Exit unconditionally, i.e. without performing the
5373	 * #UD checks (see above), is functionally ok because KVM doesn't allow
5374	 * L1 to run L2 without CR4.VMXE=0, and because KVM never modifies L2's
5375	 * CR0 or CR4, i.e. it's L2's responsibility to emulate #UDs that are
5376	 * missed by hardware due to shadowing CR0 and/or CR4.
5377	 */
5378	if (vmx_get_cpl(vcpu)) {
5379		kvm_inject_gp(vcpu, 0);
5380		return 1;
5381	}
5382
5383	if (vmx->nested.vmxon)
5384		return nested_vmx_fail(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
5385
5386	/*
5387	 * Invalid CR0/CR4 generates #GP.  These checks are performed if and
5388	 * only if the vCPU isn't already in VMX operation, i.e. effectively
5389	 * have lower priority than the VM-Fail above.
5390	 */
5391	if (!nested_host_cr0_valid(vcpu, kvm_read_cr0(vcpu)) ||
5392	    !nested_host_cr4_valid(vcpu, kvm_read_cr4(vcpu))) {
5393		kvm_inject_gp(vcpu, 0);
5394		return 1;
5395	}
5396
5397	if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
5398			!= VMXON_NEEDED_FEATURES) {
5399		kvm_inject_gp(vcpu, 0);
5400		return 1;
5401	}
5402
5403	if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret))
5404		return ret;
5405
5406	/*
5407	 * SDM 3: 24.11.5
5408	 * The first 4 bytes of VMXON region contain the supported
5409	 * VMCS revision identifier
5410	 *
5411	 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case;
5412	 * which replaces physical address width with 32
5413	 */
5414	if (!page_address_valid(vcpu, vmptr))
5415		return nested_vmx_failInvalid(vcpu);
5416
5417	if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) ||
5418	    revision != VMCS12_REVISION)
5419		return nested_vmx_failInvalid(vcpu);
5420
5421	vmx->nested.vmxon_ptr = vmptr;
5422	ret = enter_vmx_operation(vcpu);
5423	if (ret)
5424		return ret;
5425
5426	return nested_vmx_succeed(vcpu);
5427}
5428
5429static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu)
5430{
5431	struct vcpu_vmx *vmx = to_vmx(vcpu);
5432
5433	if (vmx->nested.current_vmptr == INVALID_GPA)
5434		return;
5435
5436	copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu));
5437
5438	if (enable_shadow_vmcs) {
5439		/* copy to memory all shadowed fields in case
5440		   they were modified */
5441		copy_shadow_to_vmcs12(vmx);
5442		vmx_disable_shadow_vmcs(vmx);
5443	}
5444	vmx->nested.posted_intr_nv = -1;
5445
5446	/* Flush VMCS12 to guest memory */
5447	kvm_vcpu_write_guest_page(vcpu,
5448				  vmx->nested.current_vmptr >> PAGE_SHIFT,
5449				  vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
5450
5451	kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
5452
5453	vmx->nested.current_vmptr = INVALID_GPA;
5454}
5455
5456/* Emulate the VMXOFF instruction */
5457static int handle_vmxoff(struct kvm_vcpu *vcpu)
5458{
5459	if (!nested_vmx_check_permission(vcpu))
5460		return 1;
5461
5462	free_nested(vcpu);
5463
5464	if (kvm_apic_has_pending_init_or_sipi(vcpu))
5465		kvm_make_request(KVM_REQ_EVENT, vcpu);
5466
5467	return nested_vmx_succeed(vcpu);
5468}
5469
5470/* Emulate the VMCLEAR instruction */
5471static int handle_vmclear(struct kvm_vcpu *vcpu)
5472{
5473	struct vcpu_vmx *vmx = to_vmx(vcpu);
5474	u32 zero = 0;
5475	gpa_t vmptr;
5476	int r;
5477
5478	if (!nested_vmx_check_permission(vcpu))
5479		return 1;
5480
5481	if (nested_vmx_get_vmptr(vcpu, &vmptr, &r))
5482		return r;
5483
5484	if (!page_address_valid(vcpu, vmptr))
5485		return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS);
5486
5487	if (vmptr == vmx->nested.vmxon_ptr)
5488		return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER);
5489
5490	if (likely(!nested_evmcs_handle_vmclear(vcpu, vmptr))) {
5491		if (vmptr == vmx->nested.current_vmptr)
5492			nested_release_vmcs12(vcpu);
5493
5494		/*
5495		 * Silently ignore memory errors on VMCLEAR, Intel's pseudocode
5496		 * for VMCLEAR includes a "ensure that data for VMCS referenced
5497		 * by the operand is in memory" clause that guards writes to
5498		 * memory, i.e. doing nothing for I/O is architecturally valid.
5499		 *
5500		 * FIXME: Suppress failures if and only if no memslot is found,
5501		 * i.e. exit to userspace if __copy_to_user() fails.
5502		 */
5503		(void)kvm_vcpu_write_guest(vcpu,
5504					   vmptr + offsetof(struct vmcs12,
5505							    launch_state),
5506					   &zero, sizeof(zero));
5507	}
5508
5509	return nested_vmx_succeed(vcpu);
5510}
5511
5512/* Emulate the VMLAUNCH instruction */
5513static int handle_vmlaunch(struct kvm_vcpu *vcpu)
5514{
5515	return nested_vmx_run(vcpu, true);
5516}
5517
5518/* Emulate the VMRESUME instruction */
5519static int handle_vmresume(struct kvm_vcpu *vcpu)
5520{
5521
5522	return nested_vmx_run(vcpu, false);
5523}
5524
5525static int handle_vmread(struct kvm_vcpu *vcpu)
5526{
5527	struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
5528						    : get_vmcs12(vcpu);
5529	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
5530	u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5531	struct vcpu_vmx *vmx = to_vmx(vcpu);
5532	struct x86_exception e;
5533	unsigned long field;
5534	u64 value;
5535	gva_t gva = 0;
5536	short offset;
5537	int len, r;
5538
5539	if (!nested_vmx_check_permission(vcpu))
5540		return 1;
5541
5542	/* Decode instruction info and find the field to read */
5543	field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf));
5544
5545	if (!nested_vmx_is_evmptr12_valid(vmx)) {
5546		/*
5547		 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA,
5548		 * any VMREAD sets the ALU flags for VMfailInvalid.
5549		 */
5550		if (vmx->nested.current_vmptr == INVALID_GPA ||
5551		    (is_guest_mode(vcpu) &&
5552		     get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA))
5553			return nested_vmx_failInvalid(vcpu);
5554
5555		offset = get_vmcs12_field_offset(field);
5556		if (offset < 0)
5557			return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
5558
5559		if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field))
5560			copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
5561
5562		/* Read the field, zero-extended to a u64 value */
5563		value = vmcs12_read_any(vmcs12, field, offset);
5564	} else {
5565		/*
5566		 * Hyper-V TLFS (as of 6.0b) explicitly states, that while an
5567		 * enlightened VMCS is active VMREAD/VMWRITE instructions are
5568		 * unsupported. Unfortunately, certain versions of Windows 11
5569		 * don't comply with this requirement which is not enforced in
5570		 * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a
5571		 * workaround, as misbehaving guests will panic on VM-Fail.
5572		 * Note, enlightened VMCS is incompatible with shadow VMCS so
5573		 * all VMREADs from L2 should go to L1.
5574		 */
5575		if (WARN_ON_ONCE(is_guest_mode(vcpu)))
5576			return nested_vmx_failInvalid(vcpu);
5577
5578		offset = evmcs_field_offset(field, NULL);
5579		if (offset < 0)
5580			return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
5581
5582		/* Read the field, zero-extended to a u64 value */
5583		value = evmcs_read_any(nested_vmx_evmcs(vmx), field, offset);
5584	}
5585
5586	/*
5587	 * Now copy part of this value to register or memory, as requested.
5588	 * Note that the number of bits actually copied is 32 or 64 depending
5589	 * on the guest's mode (32 or 64 bit), not on the given field's length.
5590	 */
5591	if (instr_info & BIT(10)) {
5592		kvm_register_write(vcpu, (((instr_info) >> 3) & 0xf), value);
5593	} else {
5594		len = is_64_bit_mode(vcpu) ? 8 : 4;
5595		if (get_vmx_mem_address(vcpu, exit_qualification,
5596					instr_info, true, len, &gva))
5597			return 1;
5598		/* _system ok, nested_vmx_check_permission has verified cpl=0 */
5599		r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e);
5600		if (r != X86EMUL_CONTINUE)
5601			return kvm_handle_memory_failure(vcpu, r, &e);
5602	}
5603
5604	return nested_vmx_succeed(vcpu);
5605}
5606
5607static bool is_shadow_field_rw(unsigned long field)
5608{
5609	switch (field) {
5610#define SHADOW_FIELD_RW(x, y) case x:
5611#include "vmcs_shadow_fields.h"
5612		return true;
5613	default:
5614		break;
5615	}
5616	return false;
5617}
5618
5619static bool is_shadow_field_ro(unsigned long field)
5620{
5621	switch (field) {
5622#define SHADOW_FIELD_RO(x, y) case x:
5623#include "vmcs_shadow_fields.h"
5624		return true;
5625	default:
5626		break;
5627	}
5628	return false;
5629}
5630
5631static int handle_vmwrite(struct kvm_vcpu *vcpu)
5632{
5633	struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
5634						    : get_vmcs12(vcpu);
5635	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
5636	u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5637	struct vcpu_vmx *vmx = to_vmx(vcpu);
5638	struct x86_exception e;
5639	unsigned long field;
5640	short offset;
5641	gva_t gva;
5642	int len, r;
5643
5644	/*
5645	 * The value to write might be 32 or 64 bits, depending on L1's long
5646	 * mode, and eventually we need to write that into a field of several
5647	 * possible lengths. The code below first zero-extends the value to 64
5648	 * bit (value), and then copies only the appropriate number of
5649	 * bits into the vmcs12 field.
5650	 */
5651	u64 value = 0;
5652
5653	if (!nested_vmx_check_permission(vcpu))
5654		return 1;
5655
5656	/*
5657	 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA,
5658	 * any VMWRITE sets the ALU flags for VMfailInvalid.
5659	 */
5660	if (vmx->nested.current_vmptr == INVALID_GPA ||
5661	    (is_guest_mode(vcpu) &&
5662	     get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA))
5663		return nested_vmx_failInvalid(vcpu);
5664
5665	if (instr_info & BIT(10))
5666		value = kvm_register_read(vcpu, (((instr_info) >> 3) & 0xf));
5667	else {
5668		len = is_64_bit_mode(vcpu) ? 8 : 4;
5669		if (get_vmx_mem_address(vcpu, exit_qualification,
5670					instr_info, false, len, &gva))
5671			return 1;
5672		r = kvm_read_guest_virt(vcpu, gva, &value, len, &e);
5673		if (r != X86EMUL_CONTINUE)
5674			return kvm_handle_memory_failure(vcpu, r, &e);
5675	}
5676
5677	field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf));
5678
5679	offset = get_vmcs12_field_offset(field);
5680	if (offset < 0)
5681		return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
5682
5683	/*
5684	 * If the vCPU supports "VMWRITE to any supported field in the
5685	 * VMCS," then the "read-only" fields are actually read/write.
5686	 */
5687	if (vmcs_field_readonly(field) &&
5688	    !nested_cpu_has_vmwrite_any_field(vcpu))
5689		return nested_vmx_fail(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
5690
5691	/*
5692	 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties
5693	 * vmcs12, else we may crush a field or consume a stale value.
5694	 */
5695	if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field))
5696		copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
5697
5698	/*
5699	 * Some Intel CPUs intentionally drop the reserved bits of the AR byte
5700	 * fields on VMWRITE.  Emulate this behavior to ensure consistent KVM
5701	 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE
5702	 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD
5703	 * from L1 will return a different value than VMREAD from L2 (L1 sees
5704	 * the stripped down value, L2 sees the full value as stored by KVM).
5705	 */
5706	if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES)
5707		value &= 0x1f0ff;
5708
5709	vmcs12_write_any(vmcs12, field, offset, value);
5710
5711	/*
5712	 * Do not track vmcs12 dirty-state if in guest-mode as we actually
5713	 * dirty shadow vmcs12 instead of vmcs12.  Fields that can be updated
5714	 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't
5715	 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path.
5716	 */
5717	if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) {
5718		/*
5719		 * L1 can read these fields without exiting, ensure the
5720		 * shadow VMCS is up-to-date.
5721		 */
5722		if (enable_shadow_vmcs && is_shadow_field_ro(field)) {
5723			preempt_disable();
5724			vmcs_load(vmx->vmcs01.shadow_vmcs);
5725
5726			__vmcs_writel(field, value);
5727
5728			vmcs_clear(vmx->vmcs01.shadow_vmcs);
5729			vmcs_load(vmx->loaded_vmcs->vmcs);
5730			preempt_enable();
5731		}
5732		vmx->nested.dirty_vmcs12 = true;
5733	}
5734
5735	return nested_vmx_succeed(vcpu);
5736}
5737
5738static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
5739{
5740	vmx->nested.current_vmptr = vmptr;
5741	if (enable_shadow_vmcs) {
5742		secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
5743		vmcs_write64(VMCS_LINK_POINTER,
5744			     __pa(vmx->vmcs01.shadow_vmcs));
5745		vmx->nested.need_vmcs12_to_shadow_sync = true;
5746	}
5747	vmx->nested.dirty_vmcs12 = true;
5748	vmx->nested.force_msr_bitmap_recalc = true;
5749}
5750
5751/* Emulate the VMPTRLD instruction */
5752static int handle_vmptrld(struct kvm_vcpu *vcpu)
5753{
5754	struct vcpu_vmx *vmx = to_vmx(vcpu);
5755	gpa_t vmptr;
5756	int r;
5757
5758	if (!nested_vmx_check_permission(vcpu))
5759		return 1;
5760
5761	if (nested_vmx_get_vmptr(vcpu, &vmptr, &r))
5762		return r;
5763
5764	if (!page_address_valid(vcpu, vmptr))
5765		return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS);
5766
5767	if (vmptr == vmx->nested.vmxon_ptr)
5768		return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER);
5769
5770	/* Forbid normal VMPTRLD if Enlightened version was used */
5771	if (nested_vmx_is_evmptr12_valid(vmx))
5772		return 1;
5773
5774	if (vmx->nested.current_vmptr != vmptr) {
5775		struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache;
5776		struct vmcs_hdr hdr;
5777
5778		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) {
5779			/*
5780			 * Reads from an unbacked page return all 1s,
5781			 * which means that the 32 bits located at the
5782			 * given physical address won't match the required
5783			 * VMCS12_REVISION identifier.
5784			 */
5785			return nested_vmx_fail(vcpu,
5786				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5787		}
5788
5789		if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr,
5790						 offsetof(struct vmcs12, hdr),
5791						 sizeof(hdr))) {
5792			return nested_vmx_fail(vcpu,
5793				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5794		}
5795
5796		if (hdr.revision_id != VMCS12_REVISION ||
5797		    (hdr.shadow_vmcs &&
5798		     !nested_cpu_has_vmx_shadow_vmcs(vcpu))) {
5799			return nested_vmx_fail(vcpu,
5800				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5801		}
5802
5803		nested_release_vmcs12(vcpu);
5804
5805		/*
5806		 * Load VMCS12 from guest memory since it is not already
5807		 * cached.
5808		 */
5809		if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12,
5810					  VMCS12_SIZE)) {
5811			return nested_vmx_fail(vcpu,
5812				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5813		}
5814
5815		set_current_vmptr(vmx, vmptr);
5816	}
5817
5818	return nested_vmx_succeed(vcpu);
5819}
5820
5821/* Emulate the VMPTRST instruction */
5822static int handle_vmptrst(struct kvm_vcpu *vcpu)
5823{
5824	unsigned long exit_qual = vmx_get_exit_qual(vcpu);
5825	u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5826	gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
5827	struct x86_exception e;
5828	gva_t gva;
5829	int r;
5830
5831	if (!nested_vmx_check_permission(vcpu))
5832		return 1;
5833
5834	if (unlikely(nested_vmx_is_evmptr12_valid(to_vmx(vcpu))))
5835		return 1;
5836
5837	if (get_vmx_mem_address(vcpu, exit_qual, instr_info,
5838				true, sizeof(gpa_t), &gva))
5839		return 1;
5840	/* *_system ok, nested_vmx_check_permission has verified cpl=0 */
5841	r = kvm_write_guest_virt_system(vcpu, gva, (void *)&current_vmptr,
5842					sizeof(gpa_t), &e);
5843	if (r != X86EMUL_CONTINUE)
5844		return kvm_handle_memory_failure(vcpu, r, &e);
5845
5846	return nested_vmx_succeed(vcpu);
5847}
5848
5849/* Emulate the INVEPT instruction */
5850static int handle_invept(struct kvm_vcpu *vcpu)
5851{
5852	struct vcpu_vmx *vmx = to_vmx(vcpu);
5853	u32 vmx_instruction_info, types;
5854	unsigned long type, roots_to_free;
5855	struct kvm_mmu *mmu;
5856	gva_t gva;
5857	struct x86_exception e;
5858	struct {
5859		u64 eptp, gpa;
5860	} operand;
5861	int i, r, gpr_index;
5862
5863	if (!(vmx->nested.msrs.secondary_ctls_high &
5864	      SECONDARY_EXEC_ENABLE_EPT) ||
5865	    !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) {
5866		kvm_queue_exception(vcpu, UD_VECTOR);
5867		return 1;
5868	}
5869
5870	if (!nested_vmx_check_permission(vcpu))
5871		return 1;
5872
5873	vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5874	gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info);
5875	type = kvm_register_read(vcpu, gpr_index);
5876
5877	types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
5878
5879	if (type >= 32 || !(types & (1 << type)))
5880		return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5881
5882	/* According to the Intel VMX instruction reference, the memory
5883	 * operand is read even if it isn't needed (e.g., for type==global)
5884	 */
5885	if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
5886			vmx_instruction_info, false, sizeof(operand), &gva))
5887		return 1;
5888	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
5889	if (r != X86EMUL_CONTINUE)
5890		return kvm_handle_memory_failure(vcpu, r, &e);
5891
5892	/*
5893	 * Nested EPT roots are always held through guest_mmu,
5894	 * not root_mmu.
5895	 */
5896	mmu = &vcpu->arch.guest_mmu;
5897
5898	switch (type) {
5899	case VMX_EPT_EXTENT_CONTEXT:
5900		if (!nested_vmx_check_eptp(vcpu, operand.eptp))
5901			return nested_vmx_fail(vcpu,
5902				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5903
5904		roots_to_free = 0;
5905		if (nested_ept_root_matches(mmu->root.hpa, mmu->root.pgd,
5906					    operand.eptp))
5907			roots_to_free |= KVM_MMU_ROOT_CURRENT;
5908
5909		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
5910			if (nested_ept_root_matches(mmu->prev_roots[i].hpa,
5911						    mmu->prev_roots[i].pgd,
5912						    operand.eptp))
5913				roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
5914		}
5915		break;
5916	case VMX_EPT_EXTENT_GLOBAL:
5917		roots_to_free = KVM_MMU_ROOTS_ALL;
5918		break;
5919	default:
5920		BUG();
5921		break;
5922	}
5923
5924	if (roots_to_free)
5925		kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);
5926
5927	return nested_vmx_succeed(vcpu);
5928}
5929
5930static int handle_invvpid(struct kvm_vcpu *vcpu)
5931{
5932	struct vcpu_vmx *vmx = to_vmx(vcpu);
5933	u32 vmx_instruction_info;
5934	unsigned long type, types;
5935	gva_t gva;
5936	struct x86_exception e;
5937	struct {
5938		u64 vpid;
5939		u64 gla;
5940	} operand;
5941	u16 vpid02;
5942	int r, gpr_index;
5943
5944	if (!(vmx->nested.msrs.secondary_ctls_high &
5945	      SECONDARY_EXEC_ENABLE_VPID) ||
5946			!(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) {
5947		kvm_queue_exception(vcpu, UD_VECTOR);
5948		return 1;
5949	}
5950
5951	if (!nested_vmx_check_permission(vcpu))
5952		return 1;
5953
5954	vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5955	gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info);
5956	type = kvm_register_read(vcpu, gpr_index);
5957
5958	types = (vmx->nested.msrs.vpid_caps &
5959			VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
5960
5961	if (type >= 32 || !(types & (1 << type)))
5962		return nested_vmx_fail(vcpu,
5963			VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5964
5965	/* according to the intel vmx instruction reference, the memory
5966	 * operand is read even if it isn't needed (e.g., for type==global)
5967	 */
5968	if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
5969			vmx_instruction_info, false, sizeof(operand), &gva))
5970		return 1;
5971	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
5972	if (r != X86EMUL_CONTINUE)
5973		return kvm_handle_memory_failure(vcpu, r, &e);
5974
5975	if (operand.vpid >> 16)
5976		return nested_vmx_fail(vcpu,
5977			VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5978
5979	/*
5980	 * Always flush the effective vpid02, i.e. never flush the current VPID
5981	 * and never explicitly flush vpid01.  INVVPID targets a VPID, not a
5982	 * VMCS, and so whether or not the current vmcs12 has VPID enabled is
5983	 * irrelevant (and there may not be a loaded vmcs12).
5984	 */
5985	vpid02 = nested_get_vpid02(vcpu);
5986	switch (type) {
5987	case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
5988		/*
5989		 * LAM doesn't apply to addresses that are inputs to TLB
5990		 * invalidation.
5991		 */
5992		if (!operand.vpid ||
5993		    is_noncanonical_invlpg_address(operand.gla, vcpu))
5994			return nested_vmx_fail(vcpu,
5995				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5996		vpid_sync_vcpu_addr(vpid02, operand.gla);
5997		break;
5998	case VMX_VPID_EXTENT_SINGLE_CONTEXT:
5999	case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
6000		if (!operand.vpid)
6001			return nested_vmx_fail(vcpu,
6002				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
6003		vpid_sync_context(vpid02);
6004		break;
6005	case VMX_VPID_EXTENT_ALL_CONTEXT:
6006		vpid_sync_context(vpid02);
6007		break;
6008	default:
6009		WARN_ON_ONCE(1);
6010		return kvm_skip_emulated_instruction(vcpu);
6011	}
6012
6013	/*
6014	 * Sync the shadow page tables if EPT is disabled, L1 is invalidating
6015	 * linear mappings for L2 (tagged with L2's VPID).  Free all guest
6016	 * roots as VPIDs are not tracked in the MMU role.
6017	 *
6018	 * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share
6019	 * an MMU when EPT is disabled.
6020	 *
6021	 * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR.
6022	 */
6023	if (!enable_ept)
6024		kvm_mmu_free_guest_mode_roots(vcpu->kvm, &vcpu->arch.root_mmu);
6025
6026	return nested_vmx_succeed(vcpu);
6027}
6028
6029static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
6030				     struct vmcs12 *vmcs12)
6031{
6032	u32 index = kvm_rcx_read(vcpu);
6033	u64 new_eptp;
6034
6035	if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12)))
6036		return 1;
6037	if (index >= VMFUNC_EPTP_ENTRIES)
6038		return 1;
6039
6040	if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT,
6041				     &new_eptp, index * 8, 8))
6042		return 1;
6043
6044	/*
6045	 * If the (L2) guest does a vmfunc to the currently
6046	 * active ept pointer, we don't have to do anything else
6047	 */
6048	if (vmcs12->ept_pointer != new_eptp) {
6049		if (!nested_vmx_check_eptp(vcpu, new_eptp))
6050			return 1;
6051
6052		vmcs12->ept_pointer = new_eptp;
6053		nested_ept_new_eptp(vcpu);
6054
6055		if (!nested_cpu_has_vpid(vmcs12))
6056			kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
6057	}
6058
6059	return 0;
6060}
6061
6062static int handle_vmfunc(struct kvm_vcpu *vcpu)
6063{
6064	struct vcpu_vmx *vmx = to_vmx(vcpu);
6065	struct vmcs12 *vmcs12;
6066	u32 function = kvm_rax_read(vcpu);
6067
6068	/*
6069	 * VMFUNC should never execute cleanly while L1 is active; KVM supports
6070	 * VMFUNC for nested VMs, but not for L1.
6071	 */
6072	if (WARN_ON_ONCE(!is_guest_mode(vcpu))) {
6073		kvm_queue_exception(vcpu, UD_VECTOR);
6074		return 1;
6075	}
6076
6077	vmcs12 = get_vmcs12(vcpu);
6078
6079	/*
6080	 * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC
6081	 * is enabled in vmcs02 if and only if it's enabled in vmcs12.
6082	 */
6083	if (WARN_ON_ONCE((function > 63) || !nested_cpu_has_vmfunc(vmcs12))) {
6084		kvm_queue_exception(vcpu, UD_VECTOR);
6085		return 1;
6086	}
6087
6088	if (!(vmcs12->vm_function_control & BIT_ULL(function)))
6089		goto fail;
6090
6091	switch (function) {
6092	case 0:
6093		if (nested_vmx_eptp_switching(vcpu, vmcs12))
6094			goto fail;
6095		break;
6096	default:
6097		goto fail;
6098	}
6099	return kvm_skip_emulated_instruction(vcpu);
6100
6101fail:
6102	/*
6103	 * This is effectively a reflected VM-Exit, as opposed to a synthesized
6104	 * nested VM-Exit.  Pass the original exit reason, i.e. don't hardcode
6105	 * EXIT_REASON_VMFUNC as the exit reason.
6106	 */
6107	nested_vmx_vmexit(vcpu, vmx->exit_reason.full,
6108			  vmx_get_intr_info(vcpu),
6109			  vmx_get_exit_qual(vcpu));
6110	return 1;
6111}
6112
6113/*
6114 * Return true if an IO instruction with the specified port and size should cause
6115 * a VM-exit into L1.
6116 */
6117bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port,
6118				 int size)
6119{
6120	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6121	gpa_t bitmap, last_bitmap;
6122	u8 b;
6123
6124	last_bitmap = INVALID_GPA;
6125	b = -1;
6126
6127	while (size > 0) {
6128		if (port < 0x8000)
6129			bitmap = vmcs12->io_bitmap_a;
6130		else if (port < 0x10000)
6131			bitmap = vmcs12->io_bitmap_b;
6132		else
6133			return true;
6134		bitmap += (port & 0x7fff) / 8;
6135
6136		if (last_bitmap != bitmap)
6137			if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
6138				return true;
6139		if (b & (1 << (port & 7)))
6140			return true;
6141
6142		port++;
6143		size--;
6144		last_bitmap = bitmap;
6145	}
6146
6147	return false;
6148}
6149
6150static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
6151				       struct vmcs12 *vmcs12)
6152{
6153	unsigned long exit_qualification;
6154	unsigned short port;
6155	int size;
6156
6157	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
6158		return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
6159
6160	exit_qualification = vmx_get_exit_qual(vcpu);
6161
6162	port = exit_qualification >> 16;
6163	size = (exit_qualification & 7) + 1;
6164
6165	return nested_vmx_check_io_bitmaps(vcpu, port, size);
6166}
6167
6168/*
6169 * Return 1 if we should exit from L2 to L1 to handle an MSR access,
6170 * rather than handle it ourselves in L0. I.e., check whether L1 expressed
6171 * disinterest in the current event (read or write a specific MSR) by using an
6172 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
6173 */
6174static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
6175					struct vmcs12 *vmcs12,
6176					union vmx_exit_reason exit_reason)
6177{
6178	u32 msr_index = kvm_rcx_read(vcpu);
6179	gpa_t bitmap;
6180
6181	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
6182		return true;
6183
6184	/*
6185	 * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
6186	 * for the four combinations of read/write and low/high MSR numbers.
6187	 * First we need to figure out which of the four to use:
6188	 */
6189	bitmap = vmcs12->msr_bitmap;
6190	if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
6191		bitmap += 2048;
6192	if (msr_index >= 0xc0000000) {
6193		msr_index -= 0xc0000000;
6194		bitmap += 1024;
6195	}
6196
6197	/* Then read the msr_index'th bit from this bitmap: */
6198	if (msr_index < 1024*8) {
6199		unsigned char b;
6200		if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
6201			return true;
6202		return 1 & (b >> (msr_index & 7));
6203	} else
6204		return true; /* let L1 handle the wrong parameter */
6205}
6206
6207/*
6208 * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
6209 * rather than handle it ourselves in L0. I.e., check if L1 wanted to
6210 * intercept (via guest_host_mask etc.) the current event.
6211 */
6212static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
6213	struct vmcs12 *vmcs12)
6214{
6215	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
6216	int cr = exit_qualification & 15;
6217	int reg;
6218	unsigned long val;
6219
6220	switch ((exit_qualification >> 4) & 3) {
6221	case 0: /* mov to cr */
6222		reg = (exit_qualification >> 8) & 15;
6223		val = kvm_register_read(vcpu, reg);
6224		switch (cr) {
6225		case 0:
6226			if (vmcs12->cr0_guest_host_mask &
6227			    (val ^ vmcs12->cr0_read_shadow))
6228				return true;
6229			break;
6230		case 3:
6231			if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
6232				return true;
6233			break;
6234		case 4:
6235			if (vmcs12->cr4_guest_host_mask &
6236			    (vmcs12->cr4_read_shadow ^ val))
6237				return true;
6238			break;
6239		case 8:
6240			if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
6241				return true;
6242			break;
6243		}
6244		break;
6245	case 2: /* clts */
6246		if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
6247		    (vmcs12->cr0_read_shadow & X86_CR0_TS))
6248			return true;
6249		break;
6250	case 1: /* mov from cr */
6251		switch (cr) {
6252		case 3:
6253			if (vmcs12->cpu_based_vm_exec_control &
6254			    CPU_BASED_CR3_STORE_EXITING)
6255				return true;
6256			break;
6257		case 8:
6258			if (vmcs12->cpu_based_vm_exec_control &
6259			    CPU_BASED_CR8_STORE_EXITING)
6260				return true;
6261			break;
6262		}
6263		break;
6264	case 3: /* lmsw */
6265		/*
6266		 * lmsw can change bits 1..3 of cr0, and only set bit 0 of
6267		 * cr0. Other attempted changes are ignored, with no exit.
6268		 */
6269		val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
6270		if (vmcs12->cr0_guest_host_mask & 0xe &
6271		    (val ^ vmcs12->cr0_read_shadow))
6272			return true;
6273		if ((vmcs12->cr0_guest_host_mask & 0x1) &&
6274		    !(vmcs12->cr0_read_shadow & 0x1) &&
6275		    (val & 0x1))
6276			return true;
6277		break;
6278	}
6279	return false;
6280}
6281
6282static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu,
6283					  struct vmcs12 *vmcs12)
6284{
6285	u32 encls_leaf;
6286
6287	if (!guest_cpuid_has(vcpu, X86_FEATURE_SGX) ||
6288	    !nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING))
6289		return false;
6290
6291	encls_leaf = kvm_rax_read(vcpu);
6292	if (encls_leaf > 62)
6293		encls_leaf = 63;
6294	return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf);
6295}
6296
6297static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu,
6298	struct vmcs12 *vmcs12, gpa_t bitmap)
6299{
6300	u32 vmx_instruction_info;
6301	unsigned long field;
6302	u8 b;
6303
6304	if (!nested_cpu_has_shadow_vmcs(vmcs12))
6305		return true;
6306
6307	/* Decode instruction info and find the field to access */
6308	vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
6309	field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
6310
6311	/* Out-of-range fields always cause a VM exit from L2 to L1 */
6312	if (field >> 15)
6313		return true;
6314
6315	if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1))
6316		return true;
6317
6318	return 1 & (b >> (field & 7));
6319}
6320
6321static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12)
6322{
6323	u32 entry_intr_info = vmcs12->vm_entry_intr_info_field;
6324
6325	if (nested_cpu_has_mtf(vmcs12))
6326		return true;
6327
6328	/*
6329	 * An MTF VM-exit may be injected into the guest by setting the
6330	 * interruption-type to 7 (other event) and the vector field to 0. Such
6331	 * is the case regardless of the 'monitor trap flag' VM-execution
6332	 * control.
6333	 */
6334	return entry_intr_info == (INTR_INFO_VALID_MASK
6335				   | INTR_TYPE_OTHER_EVENT);
6336}
6337
6338/*
6339 * Return true if L0 wants to handle an exit from L2 regardless of whether or not
6340 * L1 wants the exit.  Only call this when in is_guest_mode (L2).
6341 */
6342static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
6343				     union vmx_exit_reason exit_reason)
6344{
6345	u32 intr_info;
6346
6347	switch ((u16)exit_reason.basic) {
6348	case EXIT_REASON_EXCEPTION_NMI:
6349		intr_info = vmx_get_intr_info(vcpu);
6350		if (is_nmi(intr_info))
6351			return true;
6352		else if (is_page_fault(intr_info))
6353			return vcpu->arch.apf.host_apf_flags ||
6354			       vmx_need_pf_intercept(vcpu);
6355		else if (is_debug(intr_info) &&
6356			 vcpu->guest_debug &
6357			 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
6358			return true;
6359		else if (is_breakpoint(intr_info) &&
6360			 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
6361			return true;
6362		else if (is_alignment_check(intr_info) &&
6363			 !vmx_guest_inject_ac(vcpu))
6364			return true;
6365		else if (is_ve_fault(intr_info))
6366			return true;
6367		return false;
6368	case EXIT_REASON_EXTERNAL_INTERRUPT:
6369		return true;
6370	case EXIT_REASON_MCE_DURING_VMENTRY:
6371		return true;
6372	case EXIT_REASON_EPT_VIOLATION:
6373		/*
6374		 * L0 always deals with the EPT violation. If nested EPT is
6375		 * used, and the nested mmu code discovers that the address is
6376		 * missing in the guest EPT table (EPT12), the EPT violation
6377		 * will be injected with nested_ept_inject_page_fault()
6378		 */
6379		return true;
6380	case EXIT_REASON_EPT_MISCONFIG:
6381		/*
6382		 * L2 never uses directly L1's EPT, but rather L0's own EPT
6383		 * table (shadow on EPT) or a merged EPT table that L0 built
6384		 * (EPT on EPT). So any problems with the structure of the
6385		 * table is L0's fault.
6386		 */
6387		return true;
6388	case EXIT_REASON_PREEMPTION_TIMER:
6389		return true;
6390	case EXIT_REASON_PML_FULL:
6391		/*
6392		 * PML is emulated for an L1 VMM and should never be enabled in
6393		 * vmcs02, always "handle" PML_FULL by exiting to userspace.
6394		 */
6395		return true;
6396	case EXIT_REASON_VMFUNC:
6397		/* VM functions are emulated through L2->L0 vmexits. */
6398		return true;
6399	case EXIT_REASON_BUS_LOCK:
6400		/*
6401		 * At present, bus lock VM exit is never exposed to L1.
6402		 * Handle L2's bus locks in L0 directly.
6403		 */
6404		return true;
6405#ifdef CONFIG_KVM_HYPERV
6406	case EXIT_REASON_VMCALL:
6407		/* Hyper-V L2 TLB flush hypercall is handled by L0 */
6408		return guest_hv_cpuid_has_l2_tlb_flush(vcpu) &&
6409			nested_evmcs_l2_tlb_flush_enabled(vcpu) &&
6410			kvm_hv_is_tlb_flush_hcall(vcpu);
6411#endif
6412	default:
6413		break;
6414	}
6415	return false;
6416}
6417
6418/*
6419 * Return 1 if L1 wants to intercept an exit from L2.  Only call this when in
6420 * is_guest_mode (L2).
6421 */
6422static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu,
6423				     union vmx_exit_reason exit_reason)
6424{
6425	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6426	u32 intr_info;
6427
6428	switch ((u16)exit_reason.basic) {
6429	case EXIT_REASON_EXCEPTION_NMI:
6430		intr_info = vmx_get_intr_info(vcpu);
6431		if (is_nmi(intr_info))
6432			return true;
6433		else if (is_page_fault(intr_info))
6434			return true;
6435		return vmcs12->exception_bitmap &
6436				(1u << (intr_info & INTR_INFO_VECTOR_MASK));
6437	case EXIT_REASON_EXTERNAL_INTERRUPT:
6438		return nested_exit_on_intr(vcpu);
6439	case EXIT_REASON_TRIPLE_FAULT:
6440		return true;
6441	case EXIT_REASON_INTERRUPT_WINDOW:
6442		return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING);
6443	case EXIT_REASON_NMI_WINDOW:
6444		return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING);
6445	case EXIT_REASON_TASK_SWITCH:
6446		return true;
6447	case EXIT_REASON_CPUID:
6448		return true;
6449	case EXIT_REASON_HLT:
6450		return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
6451	case EXIT_REASON_INVD:
6452		return true;
6453	case EXIT_REASON_INVLPG:
6454		return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
6455	case EXIT_REASON_RDPMC:
6456		return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
6457	case EXIT_REASON_RDRAND:
6458		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING);
6459	case EXIT_REASON_RDSEED:
6460		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING);
6461	case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
6462		return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
6463	case EXIT_REASON_VMREAD:
6464		return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
6465			vmcs12->vmread_bitmap);
6466	case EXIT_REASON_VMWRITE:
6467		return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
6468			vmcs12->vmwrite_bitmap);
6469	case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
6470	case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
6471	case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME:
6472	case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
6473	case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
6474		/*
6475		 * VMX instructions trap unconditionally. This allows L1 to
6476		 * emulate them for its L2 guest, i.e., allows 3-level nesting!
6477		 */
6478		return true;
6479	case EXIT_REASON_CR_ACCESS:
6480		return nested_vmx_exit_handled_cr(vcpu, vmcs12);
6481	case EXIT_REASON_DR_ACCESS:
6482		return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
6483	case EXIT_REASON_IO_INSTRUCTION:
6484		return nested_vmx_exit_handled_io(vcpu, vmcs12);
6485	case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
6486		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
6487	case EXIT_REASON_MSR_READ:
6488	case EXIT_REASON_MSR_WRITE:
6489		return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
6490	case EXIT_REASON_INVALID_STATE:
6491		return true;
6492	case EXIT_REASON_MWAIT_INSTRUCTION:
6493		return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
6494	case EXIT_REASON_MONITOR_TRAP_FLAG:
6495		return nested_vmx_exit_handled_mtf(vmcs12);
6496	case EXIT_REASON_MONITOR_INSTRUCTION:
6497		return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
6498	case EXIT_REASON_PAUSE_INSTRUCTION:
6499		return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
6500			nested_cpu_has2(vmcs12,
6501				SECONDARY_EXEC_PAUSE_LOOP_EXITING);
6502	case EXIT_REASON_MCE_DURING_VMENTRY:
6503		return true;
6504	case EXIT_REASON_TPR_BELOW_THRESHOLD:
6505		return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
6506	case EXIT_REASON_APIC_ACCESS:
6507	case EXIT_REASON_APIC_WRITE:
6508	case EXIT_REASON_EOI_INDUCED:
6509		/*
6510		 * The controls for "virtualize APIC accesses," "APIC-
6511		 * register virtualization," and "virtual-interrupt
6512		 * delivery" only come from vmcs12.
6513		 */
6514		return true;
6515	case EXIT_REASON_INVPCID:
6516		return
6517			nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) &&
6518			nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
6519	case EXIT_REASON_WBINVD:
6520		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
6521	case EXIT_REASON_XSETBV:
6522		return true;
6523	case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
6524		/*
6525		 * This should never happen, since it is not possible to
6526		 * set XSS to a non-zero value---neither in L1 nor in L2.
6527		 * If if it were, XSS would have to be checked against
6528		 * the XSS exit bitmap in vmcs12.
6529		 */
6530		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_XSAVES);
6531	case EXIT_REASON_UMWAIT:
6532	case EXIT_REASON_TPAUSE:
6533		return nested_cpu_has2(vmcs12,
6534			SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE);
6535	case EXIT_REASON_ENCLS:
6536		return nested_vmx_exit_handled_encls(vcpu, vmcs12);
6537	case EXIT_REASON_NOTIFY:
6538		/* Notify VM exit is not exposed to L1 */
6539		return false;
6540	default:
6541		return true;
6542	}
6543}
6544
6545/*
6546 * Conditionally reflect a VM-Exit into L1.  Returns %true if the VM-Exit was
6547 * reflected into L1.
6548 */
6549bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu)
6550{
6551	struct vcpu_vmx *vmx = to_vmx(vcpu);
6552	union vmx_exit_reason exit_reason = vmx->exit_reason;
6553	unsigned long exit_qual;
6554	u32 exit_intr_info;
6555
6556	WARN_ON_ONCE(vmx->nested.nested_run_pending);
6557
6558	/*
6559	 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM
6560	 * has already loaded L2's state.
6561	 */
6562	if (unlikely(vmx->fail)) {
6563		trace_kvm_nested_vmenter_failed(
6564			"hardware VM-instruction error: ",
6565			vmcs_read32(VM_INSTRUCTION_ERROR));
6566		exit_intr_info = 0;
6567		exit_qual = 0;
6568		goto reflect_vmexit;
6569	}
6570
6571	trace_kvm_nested_vmexit(vcpu, KVM_ISA_VMX);
6572
6573	/* If L0 (KVM) wants the exit, it trumps L1's desires. */
6574	if (nested_vmx_l0_wants_exit(vcpu, exit_reason))
6575		return false;
6576
6577	/* If L1 doesn't want the exit, handle it in L0. */
6578	if (!nested_vmx_l1_wants_exit(vcpu, exit_reason))
6579		return false;
6580
6581	/*
6582	 * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits.  For
6583	 * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would
6584	 * need to be synthesized by querying the in-kernel LAPIC, but external
6585	 * interrupts are never reflected to L1 so it's a non-issue.
6586	 */
6587	exit_intr_info = vmx_get_intr_info(vcpu);
6588	if (is_exception_with_error_code(exit_intr_info)) {
6589		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6590
6591		vmcs12->vm_exit_intr_error_code =
6592			vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
6593	}
6594	exit_qual = vmx_get_exit_qual(vcpu);
6595
6596reflect_vmexit:
6597	nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual);
6598	return true;
6599}
6600
6601static int vmx_get_nested_state(struct kvm_vcpu *vcpu,
6602				struct kvm_nested_state __user *user_kvm_nested_state,
6603				u32 user_data_size)
6604{
6605	struct vcpu_vmx *vmx;
6606	struct vmcs12 *vmcs12;
6607	struct kvm_nested_state kvm_state = {
6608		.flags = 0,
6609		.format = KVM_STATE_NESTED_FORMAT_VMX,
6610		.size = sizeof(kvm_state),
6611		.hdr.vmx.flags = 0,
6612		.hdr.vmx.vmxon_pa = INVALID_GPA,
6613		.hdr.vmx.vmcs12_pa = INVALID_GPA,
6614		.hdr.vmx.preemption_timer_deadline = 0,
6615	};
6616	struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
6617		&user_kvm_nested_state->data.vmx[0];
6618
6619	if (!vcpu)
6620		return kvm_state.size + sizeof(*user_vmx_nested_state);
6621
6622	vmx = to_vmx(vcpu);
6623	vmcs12 = get_vmcs12(vcpu);
6624
6625	if (guest_can_use(vcpu, X86_FEATURE_VMX) &&
6626	    (vmx->nested.vmxon || vmx->nested.smm.vmxon)) {
6627		kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr;
6628		kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr;
6629
6630		if (vmx_has_valid_vmcs12(vcpu)) {
6631			kvm_state.size += sizeof(user_vmx_nested_state->vmcs12);
6632
6633			/* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */
6634			if (nested_vmx_is_evmptr12_set(vmx))
6635				kvm_state.flags |= KVM_STATE_NESTED_EVMCS;
6636
6637			if (is_guest_mode(vcpu) &&
6638			    nested_cpu_has_shadow_vmcs(vmcs12) &&
6639			    vmcs12->vmcs_link_pointer != INVALID_GPA)
6640				kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12);
6641		}
6642
6643		if (vmx->nested.smm.vmxon)
6644			kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON;
6645
6646		if (vmx->nested.smm.guest_mode)
6647			kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE;
6648
6649		if (is_guest_mode(vcpu)) {
6650			kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
6651
6652			if (vmx->nested.nested_run_pending)
6653				kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
6654
6655			if (vmx->nested.mtf_pending)
6656				kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING;
6657
6658			if (nested_cpu_has_preemption_timer(vmcs12) &&
6659			    vmx->nested.has_preemption_timer_deadline) {
6660				kvm_state.hdr.vmx.flags |=
6661					KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE;
6662				kvm_state.hdr.vmx.preemption_timer_deadline =
6663					vmx->nested.preemption_timer_deadline;
6664			}
6665		}
6666	}
6667
6668	if (user_data_size < kvm_state.size)
6669		goto out;
6670
6671	if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
6672		return -EFAULT;
6673
6674	if (!vmx_has_valid_vmcs12(vcpu))
6675		goto out;
6676
6677	/*
6678	 * When running L2, the authoritative vmcs12 state is in the
6679	 * vmcs02. When running L1, the authoritative vmcs12 state is
6680	 * in the shadow or enlightened vmcs linked to vmcs01, unless
6681	 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative
6682	 * vmcs12 state is in the vmcs12 already.
6683	 */
6684	if (is_guest_mode(vcpu)) {
6685		sync_vmcs02_to_vmcs12(vcpu, vmcs12);
6686		sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
6687	} else  {
6688		copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu));
6689		if (!vmx->nested.need_vmcs12_to_shadow_sync) {
6690			if (nested_vmx_is_evmptr12_valid(vmx))
6691				/*
6692				 * L1 hypervisor is not obliged to keep eVMCS
6693				 * clean fields data always up-to-date while
6694				 * not in guest mode, 'hv_clean_fields' is only
6695				 * supposed to be actual upon vmentry so we need
6696				 * to ignore it here and do full copy.
6697				 */
6698				copy_enlightened_to_vmcs12(vmx, 0);
6699			else if (enable_shadow_vmcs)
6700				copy_shadow_to_vmcs12(vmx);
6701		}
6702	}
6703
6704	BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE);
6705	BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE);
6706
6707	/*
6708	 * Copy over the full allocated size of vmcs12 rather than just the size
6709	 * of the struct.
6710	 */
6711	if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE))
6712		return -EFAULT;
6713
6714	if (nested_cpu_has_shadow_vmcs(vmcs12) &&
6715	    vmcs12->vmcs_link_pointer != INVALID_GPA) {
6716		if (copy_to_user(user_vmx_nested_state->shadow_vmcs12,
6717				 get_shadow_vmcs12(vcpu), VMCS12_SIZE))
6718			return -EFAULT;
6719	}
6720out:
6721	return kvm_state.size;
6722}
6723
6724void vmx_leave_nested(struct kvm_vcpu *vcpu)
6725{
6726	if (is_guest_mode(vcpu)) {
6727		to_vmx(vcpu)->nested.nested_run_pending = 0;
6728		nested_vmx_vmexit(vcpu, -1, 0, 0);
6729	}
6730	free_nested(vcpu);
6731}
6732
6733static int vmx_set_nested_state(struct kvm_vcpu *vcpu,
6734				struct kvm_nested_state __user *user_kvm_nested_state,
6735				struct kvm_nested_state *kvm_state)
6736{
6737	struct vcpu_vmx *vmx = to_vmx(vcpu);
6738	struct vmcs12 *vmcs12;
6739	enum vm_entry_failure_code ignored;
6740	struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
6741		&user_kvm_nested_state->data.vmx[0];
6742	int ret;
6743
6744	if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX)
6745		return -EINVAL;
6746
6747	if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) {
6748		if (kvm_state->hdr.vmx.smm.flags)
6749			return -EINVAL;
6750
6751		if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA)
6752			return -EINVAL;
6753
6754		/*
6755		 * KVM_STATE_NESTED_EVMCS used to signal that KVM should
6756		 * enable eVMCS capability on vCPU. However, since then
6757		 * code was changed such that flag signals vmcs12 should
6758		 * be copied into eVMCS in guest memory.
6759		 *
6760		 * To preserve backwards compatibility, allow user
6761		 * to set this flag even when there is no VMXON region.
6762		 */
6763		if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS)
6764			return -EINVAL;
6765	} else {
6766		if (!guest_can_use(vcpu, X86_FEATURE_VMX))
6767			return -EINVAL;
6768
6769		if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa))
6770			return -EINVAL;
6771	}
6772
6773	if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
6774	    (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
6775		return -EINVAL;
6776
6777	if (kvm_state->hdr.vmx.smm.flags &
6778	    ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON))
6779		return -EINVAL;
6780
6781	if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE)
6782		return -EINVAL;
6783
6784	/*
6785	 * SMM temporarily disables VMX, so we cannot be in guest mode,
6786	 * nor can VMLAUNCH/VMRESUME be pending.  Outside SMM, SMM flags
6787	 * must be zero.
6788	 */
6789	if (is_smm(vcpu) ?
6790		(kvm_state->flags &
6791		 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING))
6792		: kvm_state->hdr.vmx.smm.flags)
6793		return -EINVAL;
6794
6795	if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
6796	    !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON))
6797		return -EINVAL;
6798
6799	if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) &&
6800	    (!guest_can_use(vcpu, X86_FEATURE_VMX) ||
6801	     !vmx->nested.enlightened_vmcs_enabled))
6802			return -EINVAL;
6803
6804	vmx_leave_nested(vcpu);
6805
6806	if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA)
6807		return 0;
6808
6809	vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa;
6810	ret = enter_vmx_operation(vcpu);
6811	if (ret)
6812		return ret;
6813
6814	/* Empty 'VMXON' state is permitted if no VMCS loaded */
6815	if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) {
6816		/* See vmx_has_valid_vmcs12.  */
6817		if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) ||
6818		    (kvm_state->flags & KVM_STATE_NESTED_EVMCS) ||
6819		    (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA))
6820			return -EINVAL;
6821		else
6822			return 0;
6823	}
6824
6825	if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) {
6826		if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa ||
6827		    !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa))
6828			return -EINVAL;
6829
6830		set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa);
6831#ifdef CONFIG_KVM_HYPERV
6832	} else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) {
6833		/*
6834		 * nested_vmx_handle_enlightened_vmptrld() cannot be called
6835		 * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be
6836		 * restored yet. EVMCS will be mapped from
6837		 * nested_get_vmcs12_pages().
6838		 */
6839		vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING;
6840		kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
6841#endif
6842	} else {
6843		return -EINVAL;
6844	}
6845
6846	if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) {
6847		vmx->nested.smm.vmxon = true;
6848		vmx->nested.vmxon = false;
6849
6850		if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE)
6851			vmx->nested.smm.guest_mode = true;
6852	}
6853
6854	vmcs12 = get_vmcs12(vcpu);
6855	if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12)))
6856		return -EFAULT;
6857
6858	if (vmcs12->hdr.revision_id != VMCS12_REVISION)
6859		return -EINVAL;
6860
6861	if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
6862		return 0;
6863
6864	vmx->nested.nested_run_pending =
6865		!!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING);
6866
6867	vmx->nested.mtf_pending =
6868		!!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING);
6869
6870	ret = -EINVAL;
6871	if (nested_cpu_has_shadow_vmcs(vmcs12) &&
6872	    vmcs12->vmcs_link_pointer != INVALID_GPA) {
6873		struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu);
6874
6875		if (kvm_state->size <
6876		    sizeof(*kvm_state) +
6877		    sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12))
6878			goto error_guest_mode;
6879
6880		if (copy_from_user(shadow_vmcs12,
6881				   user_vmx_nested_state->shadow_vmcs12,
6882				   sizeof(*shadow_vmcs12))) {
6883			ret = -EFAULT;
6884			goto error_guest_mode;
6885		}
6886
6887		if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION ||
6888		    !shadow_vmcs12->hdr.shadow_vmcs)
6889			goto error_guest_mode;
6890	}
6891
6892	vmx->nested.has_preemption_timer_deadline = false;
6893	if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) {
6894		vmx->nested.has_preemption_timer_deadline = true;
6895		vmx->nested.preemption_timer_deadline =
6896			kvm_state->hdr.vmx.preemption_timer_deadline;
6897	}
6898
6899	if (nested_vmx_check_controls(vcpu, vmcs12) ||
6900	    nested_vmx_check_host_state(vcpu, vmcs12) ||
6901	    nested_vmx_check_guest_state(vcpu, vmcs12, &ignored))
6902		goto error_guest_mode;
6903
6904	vmx->nested.dirty_vmcs12 = true;
6905	vmx->nested.force_msr_bitmap_recalc = true;
6906	ret = nested_vmx_enter_non_root_mode(vcpu, false);
6907	if (ret)
6908		goto error_guest_mode;
6909
6910	if (vmx->nested.mtf_pending)
6911		kvm_make_request(KVM_REQ_EVENT, vcpu);
6912
6913	return 0;
6914
6915error_guest_mode:
6916	vmx->nested.nested_run_pending = 0;
6917	return ret;
6918}
6919
6920void nested_vmx_set_vmcs_shadowing_bitmap(void)
6921{
6922	if (enable_shadow_vmcs) {
6923		vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
6924		vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
6925	}
6926}
6927
6928/*
6929 * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6.  Undo
6930 * that madness to get the encoding for comparison.
6931 */
6932#define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10)))
6933
6934static u64 nested_vmx_calc_vmcs_enum_msr(void)
6935{
6936	/*
6937	 * Note these are the so called "index" of the VMCS field encoding, not
6938	 * the index into vmcs12.
6939	 */
6940	unsigned int max_idx, idx;
6941	int i;
6942
6943	/*
6944	 * For better or worse, KVM allows VMREAD/VMWRITE to all fields in
6945	 * vmcs12, regardless of whether or not the associated feature is
6946	 * exposed to L1.  Simply find the field with the highest index.
6947	 */
6948	max_idx = 0;
6949	for (i = 0; i < nr_vmcs12_fields; i++) {
6950		/* The vmcs12 table is very, very sparsely populated. */
6951		if (!vmcs12_field_offsets[i])
6952			continue;
6953
6954		idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i));
6955		if (idx > max_idx)
6956			max_idx = idx;
6957	}
6958
6959	return (u64)max_idx << VMCS_FIELD_INDEX_SHIFT;
6960}
6961
6962static void nested_vmx_setup_pinbased_ctls(struct vmcs_config *vmcs_conf,
6963					   struct nested_vmx_msrs *msrs)
6964{
6965	msrs->pinbased_ctls_low =
6966		PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
6967
6968	msrs->pinbased_ctls_high = vmcs_conf->pin_based_exec_ctrl;
6969	msrs->pinbased_ctls_high &=
6970		PIN_BASED_EXT_INTR_MASK |
6971		PIN_BASED_NMI_EXITING |
6972		PIN_BASED_VIRTUAL_NMIS |
6973		(enable_apicv ? PIN_BASED_POSTED_INTR : 0);
6974	msrs->pinbased_ctls_high |=
6975		PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
6976		PIN_BASED_VMX_PREEMPTION_TIMER;
6977}
6978
6979static void nested_vmx_setup_exit_ctls(struct vmcs_config *vmcs_conf,
6980				       struct nested_vmx_msrs *msrs)
6981{
6982	msrs->exit_ctls_low =
6983		VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
6984
6985	msrs->exit_ctls_high = vmcs_conf->vmexit_ctrl;
6986	msrs->exit_ctls_high &=
6987#ifdef CONFIG_X86_64
6988		VM_EXIT_HOST_ADDR_SPACE_SIZE |
6989#endif
6990		VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT |
6991		VM_EXIT_CLEAR_BNDCFGS;
6992	msrs->exit_ctls_high |=
6993		VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
6994		VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
6995		VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT |
6996		VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
6997
6998	/* We support free control of debug control saving. */
6999	msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
7000}
7001
7002static void nested_vmx_setup_entry_ctls(struct vmcs_config *vmcs_conf,
7003					struct nested_vmx_msrs *msrs)
7004{
7005	msrs->entry_ctls_low =
7006		VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
7007
7008	msrs->entry_ctls_high = vmcs_conf->vmentry_ctrl;
7009	msrs->entry_ctls_high &=
7010#ifdef CONFIG_X86_64
7011		VM_ENTRY_IA32E_MODE |
7012#endif
7013		VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS;
7014	msrs->entry_ctls_high |=
7015		(VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER |
7016		 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL);
7017
7018	/* We support free control of debug control loading. */
7019	msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
7020}
7021
7022static void nested_vmx_setup_cpubased_ctls(struct vmcs_config *vmcs_conf,
7023					   struct nested_vmx_msrs *msrs)
7024{
7025	msrs->procbased_ctls_low =
7026		CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
7027
7028	msrs->procbased_ctls_high = vmcs_conf->cpu_based_exec_ctrl;
7029	msrs->procbased_ctls_high &=
7030		CPU_BASED_INTR_WINDOW_EXITING |
7031		CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING |
7032		CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
7033		CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
7034		CPU_BASED_CR3_STORE_EXITING |
7035#ifdef CONFIG_X86_64
7036		CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
7037#endif
7038		CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
7039		CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
7040		CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
7041		CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
7042		CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
7043	/*
7044	 * We can allow some features even when not supported by the
7045	 * hardware. For example, L1 can specify an MSR bitmap - and we
7046	 * can use it to avoid exits to L1 - even when L0 runs L2
7047	 * without MSR bitmaps.
7048	 */
7049	msrs->procbased_ctls_high |=
7050		CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
7051		CPU_BASED_USE_MSR_BITMAPS;
7052
7053	/* We support free control of CR3 access interception. */
7054	msrs->procbased_ctls_low &=
7055		~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
7056}
7057
7058static void nested_vmx_setup_secondary_ctls(u32 ept_caps,
7059					    struct vmcs_config *vmcs_conf,
7060					    struct nested_vmx_msrs *msrs)
7061{
7062	msrs->secondary_ctls_low = 0;
7063
7064	msrs->secondary_ctls_high = vmcs_conf->cpu_based_2nd_exec_ctrl;
7065	msrs->secondary_ctls_high &=
7066		SECONDARY_EXEC_DESC |
7067		SECONDARY_EXEC_ENABLE_RDTSCP |
7068		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
7069		SECONDARY_EXEC_WBINVD_EXITING |
7070		SECONDARY_EXEC_APIC_REGISTER_VIRT |
7071		SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
7072		SECONDARY_EXEC_RDRAND_EXITING |
7073		SECONDARY_EXEC_ENABLE_INVPCID |
7074		SECONDARY_EXEC_ENABLE_VMFUNC |
7075		SECONDARY_EXEC_RDSEED_EXITING |
7076		SECONDARY_EXEC_ENABLE_XSAVES |
7077		SECONDARY_EXEC_TSC_SCALING |
7078		SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE;
7079
7080	/*
7081	 * We can emulate "VMCS shadowing," even if the hardware
7082	 * doesn't support it.
7083	 */
7084	msrs->secondary_ctls_high |=
7085		SECONDARY_EXEC_SHADOW_VMCS;
7086
7087	if (enable_ept) {
7088		/* nested EPT: emulate EPT also to L1 */
7089		msrs->secondary_ctls_high |=
7090			SECONDARY_EXEC_ENABLE_EPT;
7091		msrs->ept_caps =
7092			VMX_EPT_PAGE_WALK_4_BIT |
7093			VMX_EPT_PAGE_WALK_5_BIT |
7094			VMX_EPTP_WB_BIT |
7095			VMX_EPT_INVEPT_BIT |
7096			VMX_EPT_EXECUTE_ONLY_BIT;
7097
7098		msrs->ept_caps &= ept_caps;
7099		msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
7100			VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
7101			VMX_EPT_1GB_PAGE_BIT;
7102		if (enable_ept_ad_bits) {
7103			msrs->secondary_ctls_high |=
7104				SECONDARY_EXEC_ENABLE_PML;
7105			msrs->ept_caps |= VMX_EPT_AD_BIT;
7106		}
7107
7108		/*
7109		 * Advertise EPTP switching irrespective of hardware support,
7110		 * KVM emulates it in software so long as VMFUNC is supported.
7111		 */
7112		if (cpu_has_vmx_vmfunc())
7113			msrs->vmfunc_controls = VMX_VMFUNC_EPTP_SWITCHING;
7114	}
7115
7116	/*
7117	 * Old versions of KVM use the single-context version without
7118	 * checking for support, so declare that it is supported even
7119	 * though it is treated as global context.  The alternative is
7120	 * not failing the single-context invvpid, and it is worse.
7121	 */
7122	if (enable_vpid) {
7123		msrs->secondary_ctls_high |=
7124			SECONDARY_EXEC_ENABLE_VPID;
7125		msrs->vpid_caps = VMX_VPID_INVVPID_BIT |
7126			VMX_VPID_EXTENT_SUPPORTED_MASK;
7127	}
7128
7129	if (enable_unrestricted_guest)
7130		msrs->secondary_ctls_high |=
7131			SECONDARY_EXEC_UNRESTRICTED_GUEST;
7132
7133	if (flexpriority_enabled)
7134		msrs->secondary_ctls_high |=
7135			SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
7136
7137	if (enable_sgx)
7138		msrs->secondary_ctls_high |= SECONDARY_EXEC_ENCLS_EXITING;
7139}
7140
7141static void nested_vmx_setup_misc_data(struct vmcs_config *vmcs_conf,
7142				       struct nested_vmx_msrs *msrs)
7143{
7144	msrs->misc_low = (u32)vmcs_conf->misc & VMX_MISC_SAVE_EFER_LMA;
7145	msrs->misc_low |=
7146		VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
7147		VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
7148		VMX_MISC_ACTIVITY_HLT |
7149		VMX_MISC_ACTIVITY_WAIT_SIPI;
7150	msrs->misc_high = 0;
7151}
7152
7153static void nested_vmx_setup_basic(struct nested_vmx_msrs *msrs)
7154{
7155	/*
7156	 * This MSR reports some information about VMX support. We
7157	 * should return information about the VMX we emulate for the
7158	 * guest, and the VMCS structure we give it - not about the
7159	 * VMX support of the underlying hardware.
7160	 */
7161	msrs->basic = vmx_basic_encode_vmcs_info(VMCS12_REVISION, VMCS12_SIZE,
7162						 X86_MEMTYPE_WB);
 
 
 
7163
7164	msrs->basic |= VMX_BASIC_TRUE_CTLS;
7165	if (cpu_has_vmx_basic_inout())
7166		msrs->basic |= VMX_BASIC_INOUT;
7167}
7168
7169static void nested_vmx_setup_cr_fixed(struct nested_vmx_msrs *msrs)
7170{
7171	/*
7172	 * These MSRs specify bits which the guest must keep fixed on
7173	 * while L1 is in VMXON mode (in L1's root mode, or running an L2).
7174	 * We picked the standard core2 setting.
7175	 */
7176#define VMXON_CR0_ALWAYSON     (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
7177#define VMXON_CR4_ALWAYSON     X86_CR4_VMXE
7178	msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON;
7179	msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON;
7180
7181	/* These MSRs specify bits which the guest must keep fixed off. */
7182	rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1);
7183	rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1);
7184
7185	if (vmx_umip_emulated())
7186		msrs->cr4_fixed1 |= X86_CR4_UMIP;
7187}
7188
7189/*
7190 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
7191 * returned for the various VMX controls MSRs when nested VMX is enabled.
7192 * The same values should also be used to verify that vmcs12 control fields are
7193 * valid during nested entry from L1 to L2.
7194 * Each of these control msrs has a low and high 32-bit half: A low bit is on
7195 * if the corresponding bit in the (32-bit) control field *must* be on, and a
7196 * bit in the high half is on if the corresponding bit in the control field
7197 * may be on. See also vmx_control_verify().
7198 */
7199void nested_vmx_setup_ctls_msrs(struct vmcs_config *vmcs_conf, u32 ept_caps)
7200{
7201	struct nested_vmx_msrs *msrs = &vmcs_conf->nested;
7202
7203	/*
7204	 * Note that as a general rule, the high half of the MSRs (bits in
7205	 * the control fields which may be 1) should be initialized by the
7206	 * intersection of the underlying hardware's MSR (i.e., features which
7207	 * can be supported) and the list of features we want to expose -
7208	 * because they are known to be properly supported in our code.
7209	 * Also, usually, the low half of the MSRs (bits which must be 1) can
7210	 * be set to 0, meaning that L1 may turn off any of these bits. The
7211	 * reason is that if one of these bits is necessary, it will appear
7212	 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
7213	 * fields of vmcs01 and vmcs02, will turn these bits off - and
7214	 * nested_vmx_l1_wants_exit() will not pass related exits to L1.
7215	 * These rules have exceptions below.
7216	 */
7217	nested_vmx_setup_pinbased_ctls(vmcs_conf, msrs);
7218
7219	nested_vmx_setup_exit_ctls(vmcs_conf, msrs);
7220
7221	nested_vmx_setup_entry_ctls(vmcs_conf, msrs);
7222
7223	nested_vmx_setup_cpubased_ctls(vmcs_conf, msrs);
7224
7225	nested_vmx_setup_secondary_ctls(ept_caps, vmcs_conf, msrs);
7226
7227	nested_vmx_setup_misc_data(vmcs_conf, msrs);
7228
7229	nested_vmx_setup_basic(msrs);
7230
7231	nested_vmx_setup_cr_fixed(msrs);
7232
7233	msrs->vmcs_enum = nested_vmx_calc_vmcs_enum_msr();
7234}
7235
7236void nested_vmx_hardware_unsetup(void)
7237{
7238	int i;
7239
7240	if (enable_shadow_vmcs) {
7241		for (i = 0; i < VMX_BITMAP_NR; i++)
7242			free_page((unsigned long)vmx_bitmap[i]);
7243	}
7244}
7245
7246__init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
7247{
7248	int i;
7249
7250	if (!cpu_has_vmx_shadow_vmcs())
7251		enable_shadow_vmcs = 0;
7252	if (enable_shadow_vmcs) {
7253		for (i = 0; i < VMX_BITMAP_NR; i++) {
7254			/*
7255			 * The vmx_bitmap is not tied to a VM and so should
7256			 * not be charged to a memcg.
7257			 */
7258			vmx_bitmap[i] = (unsigned long *)
7259				__get_free_page(GFP_KERNEL);
7260			if (!vmx_bitmap[i]) {
7261				nested_vmx_hardware_unsetup();
7262				return -ENOMEM;
7263			}
7264		}
7265
7266		init_vmcs_shadow_fields();
7267	}
7268
7269	exit_handlers[EXIT_REASON_VMCLEAR]	= handle_vmclear;
7270	exit_handlers[EXIT_REASON_VMLAUNCH]	= handle_vmlaunch;
7271	exit_handlers[EXIT_REASON_VMPTRLD]	= handle_vmptrld;
7272	exit_handlers[EXIT_REASON_VMPTRST]	= handle_vmptrst;
7273	exit_handlers[EXIT_REASON_VMREAD]	= handle_vmread;
7274	exit_handlers[EXIT_REASON_VMRESUME]	= handle_vmresume;
7275	exit_handlers[EXIT_REASON_VMWRITE]	= handle_vmwrite;
7276	exit_handlers[EXIT_REASON_VMOFF]	= handle_vmxoff;
7277	exit_handlers[EXIT_REASON_VMON]		= handle_vmxon;
7278	exit_handlers[EXIT_REASON_INVEPT]	= handle_invept;
7279	exit_handlers[EXIT_REASON_INVVPID]	= handle_invvpid;
7280	exit_handlers[EXIT_REASON_VMFUNC]	= handle_vmfunc;
7281
7282	return 0;
7283}
7284
7285struct kvm_x86_nested_ops vmx_nested_ops = {
7286	.leave_nested = vmx_leave_nested,
7287	.is_exception_vmexit = nested_vmx_is_exception_vmexit,
7288	.check_events = vmx_check_nested_events,
7289	.has_events = vmx_has_nested_events,
7290	.triple_fault = nested_vmx_triple_fault,
7291	.get_state = vmx_get_nested_state,
7292	.set_state = vmx_set_nested_state,
7293	.get_nested_state_pages = vmx_get_nested_state_pages,
7294	.write_log_dirty = nested_vmx_write_pml_buffer,
7295#ifdef CONFIG_KVM_HYPERV
7296	.enable_evmcs = nested_enable_evmcs,
7297	.get_evmcs_version = nested_get_evmcs_version,
7298	.hv_inject_synthetic_vmexit_post_tlb_flush = vmx_hv_inject_synthetic_vmexit_post_tlb_flush,
7299#endif
7300};