Loading...
1// SPDX-License-Identifier: GPL-2.0
2#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3
4#include <linux/errno.h>
5#include <linux/smp.h>
6
7#include "x86.h"
8#include "../cpuid.h"
9#include "hyperv.h"
10#include "nested.h"
11#include "vmcs.h"
12#include "vmx.h"
13#include "trace.h"
14
15#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
16
17u64 nested_get_evmptr(struct kvm_vcpu *vcpu)
18{
19 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
20
21 if (unlikely(kvm_hv_get_assist_page(vcpu)))
22 return EVMPTR_INVALID;
23
24 if (unlikely(!hv_vcpu->vp_assist_page.enlighten_vmentry))
25 return EVMPTR_INVALID;
26
27 return hv_vcpu->vp_assist_page.current_nested_vmcs;
28}
29
30uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu)
31{
32 /*
33 * vmcs_version represents the range of supported Enlightened VMCS
34 * versions: lower 8 bits is the minimal version, higher 8 bits is the
35 * maximum supported version. KVM supports versions from 1 to
36 * KVM_EVMCS_VERSION.
37 *
38 * Note, do not check the Hyper-V is fully enabled in guest CPUID, this
39 * helper is used to _get_ the vCPU's supported CPUID.
40 */
41 if (kvm_cpu_cap_get(X86_FEATURE_VMX) &&
42 (!vcpu || to_vmx(vcpu)->nested.enlightened_vmcs_enabled))
43 return (KVM_EVMCS_VERSION << 8) | 1;
44
45 return 0;
46}
47
48enum evmcs_revision {
49 EVMCSv1_LEGACY,
50 NR_EVMCS_REVISIONS,
51};
52
53enum evmcs_ctrl_type {
54 EVMCS_EXIT_CTRLS,
55 EVMCS_ENTRY_CTRLS,
56 EVMCS_EXEC_CTRL,
57 EVMCS_2NDEXEC,
58 EVMCS_3RDEXEC,
59 EVMCS_PINCTRL,
60 EVMCS_VMFUNC,
61 NR_EVMCS_CTRLS,
62};
63
64static const u32 evmcs_supported_ctrls[NR_EVMCS_CTRLS][NR_EVMCS_REVISIONS] = {
65 [EVMCS_EXIT_CTRLS] = {
66 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMEXIT_CTRL,
67 },
68 [EVMCS_ENTRY_CTRLS] = {
69 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMENTRY_CTRL,
70 },
71 [EVMCS_EXEC_CTRL] = {
72 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_EXEC_CTRL,
73 },
74 [EVMCS_2NDEXEC] = {
75 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_2NDEXEC & ~SECONDARY_EXEC_TSC_SCALING,
76 },
77 [EVMCS_3RDEXEC] = {
78 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_3RDEXEC,
79 },
80 [EVMCS_PINCTRL] = {
81 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_PINCTRL,
82 },
83 [EVMCS_VMFUNC] = {
84 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMFUNC,
85 },
86};
87
88static u32 evmcs_get_supported_ctls(enum evmcs_ctrl_type ctrl_type)
89{
90 enum evmcs_revision evmcs_rev = EVMCSv1_LEGACY;
91
92 return evmcs_supported_ctrls[ctrl_type][evmcs_rev];
93}
94
95static bool evmcs_has_perf_global_ctrl(struct kvm_vcpu *vcpu)
96{
97 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
98
99 /*
100 * PERF_GLOBAL_CTRL has a quirk where some Windows guests may fail to
101 * boot if a PV CPUID feature flag is not also set. Treat the fields
102 * as unsupported if the flag is not set in guest CPUID. This should
103 * be called only for guest accesses, and all guest accesses should be
104 * gated on Hyper-V being enabled and initialized.
105 */
106 if (WARN_ON_ONCE(!hv_vcpu))
107 return false;
108
109 return hv_vcpu->cpuid_cache.nested_ebx & HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL;
110}
111
112void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
113{
114 u32 ctl_low = (u32)*pdata;
115 u32 ctl_high = (u32)(*pdata >> 32);
116 u32 supported_ctrls;
117
118 /*
119 * Hyper-V 2016 and 2019 try using these features even when eVMCS
120 * is enabled but there are no corresponding fields.
121 */
122 switch (msr_index) {
123 case MSR_IA32_VMX_EXIT_CTLS:
124 case MSR_IA32_VMX_TRUE_EXIT_CTLS:
125 supported_ctrls = evmcs_get_supported_ctls(EVMCS_EXIT_CTRLS);
126 if (!evmcs_has_perf_global_ctrl(vcpu))
127 supported_ctrls &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
128 ctl_high &= supported_ctrls;
129 break;
130 case MSR_IA32_VMX_ENTRY_CTLS:
131 case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
132 supported_ctrls = evmcs_get_supported_ctls(EVMCS_ENTRY_CTRLS);
133 if (!evmcs_has_perf_global_ctrl(vcpu))
134 supported_ctrls &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
135 ctl_high &= supported_ctrls;
136 break;
137 case MSR_IA32_VMX_PROCBASED_CTLS:
138 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
139 ctl_high &= evmcs_get_supported_ctls(EVMCS_EXEC_CTRL);
140 break;
141 case MSR_IA32_VMX_PROCBASED_CTLS2:
142 ctl_high &= evmcs_get_supported_ctls(EVMCS_2NDEXEC);
143 break;
144 case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
145 case MSR_IA32_VMX_PINBASED_CTLS:
146 ctl_high &= evmcs_get_supported_ctls(EVMCS_PINCTRL);
147 break;
148 case MSR_IA32_VMX_VMFUNC:
149 ctl_low &= evmcs_get_supported_ctls(EVMCS_VMFUNC);
150 break;
151 }
152
153 *pdata = ctl_low | ((u64)ctl_high << 32);
154}
155
156static bool nested_evmcs_is_valid_controls(enum evmcs_ctrl_type ctrl_type,
157 u32 val)
158{
159 return !(val & ~evmcs_get_supported_ctls(ctrl_type));
160}
161
162int nested_evmcs_check_controls(struct vmcs12 *vmcs12)
163{
164 if (CC(!nested_evmcs_is_valid_controls(EVMCS_PINCTRL,
165 vmcs12->pin_based_vm_exec_control)))
166 return -EINVAL;
167
168 if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXEC_CTRL,
169 vmcs12->cpu_based_vm_exec_control)))
170 return -EINVAL;
171
172 if (CC(!nested_evmcs_is_valid_controls(EVMCS_2NDEXEC,
173 vmcs12->secondary_vm_exec_control)))
174 return -EINVAL;
175
176 if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXIT_CTRLS,
177 vmcs12->vm_exit_controls)))
178 return -EINVAL;
179
180 if (CC(!nested_evmcs_is_valid_controls(EVMCS_ENTRY_CTRLS,
181 vmcs12->vm_entry_controls)))
182 return -EINVAL;
183
184 /*
185 * VM-Func controls are 64-bit, but KVM currently doesn't support any
186 * controls in bits 63:32, i.e. dropping those bits on the consistency
187 * check is intentional.
188 */
189 if (WARN_ON_ONCE(vmcs12->vm_function_control >> 32))
190 return -EINVAL;
191
192 if (CC(!nested_evmcs_is_valid_controls(EVMCS_VMFUNC,
193 vmcs12->vm_function_control)))
194 return -EINVAL;
195
196 return 0;
197}
198
199int nested_enable_evmcs(struct kvm_vcpu *vcpu,
200 uint16_t *vmcs_version)
201{
202 struct vcpu_vmx *vmx = to_vmx(vcpu);
203
204 vmx->nested.enlightened_vmcs_enabled = true;
205
206 if (vmcs_version)
207 *vmcs_version = nested_get_evmcs_version(vcpu);
208
209 return 0;
210}
211
212bool nested_evmcs_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu)
213{
214 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
215 struct vcpu_vmx *vmx = to_vmx(vcpu);
216 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
217
218 if (!hv_vcpu || !evmcs)
219 return false;
220
221 if (!evmcs->hv_enlightenments_control.nested_flush_hypercall)
222 return false;
223
224 return hv_vcpu->vp_assist_page.nested_control.features.directhypercall;
225}
226
227void vmx_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu)
228{
229 nested_vmx_vmexit(vcpu, HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH, 0, 0);
230}
1// SPDX-License-Identifier: GPL-2.0
2#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3
4#include <linux/errno.h>
5#include <linux/smp.h>
6
7#include "../cpuid.h"
8#include "hyperv.h"
9#include "nested.h"
10#include "vmcs.h"
11#include "vmx.h"
12#include "trace.h"
13
14#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
15
16u64 nested_get_evmptr(struct kvm_vcpu *vcpu)
17{
18 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
19
20 if (unlikely(kvm_hv_get_assist_page(vcpu)))
21 return EVMPTR_INVALID;
22
23 if (unlikely(!hv_vcpu->vp_assist_page.enlighten_vmentry))
24 return EVMPTR_INVALID;
25
26 return hv_vcpu->vp_assist_page.current_nested_vmcs;
27}
28
29uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu)
30{
31 /*
32 * vmcs_version represents the range of supported Enlightened VMCS
33 * versions: lower 8 bits is the minimal version, higher 8 bits is the
34 * maximum supported version. KVM supports versions from 1 to
35 * KVM_EVMCS_VERSION.
36 *
37 * Note, do not check the Hyper-V is fully enabled in guest CPUID, this
38 * helper is used to _get_ the vCPU's supported CPUID.
39 */
40 if (kvm_cpu_cap_get(X86_FEATURE_VMX) &&
41 (!vcpu || to_vmx(vcpu)->nested.enlightened_vmcs_enabled))
42 return (KVM_EVMCS_VERSION << 8) | 1;
43
44 return 0;
45}
46
47enum evmcs_revision {
48 EVMCSv1_LEGACY,
49 NR_EVMCS_REVISIONS,
50};
51
52enum evmcs_ctrl_type {
53 EVMCS_EXIT_CTRLS,
54 EVMCS_ENTRY_CTRLS,
55 EVMCS_EXEC_CTRL,
56 EVMCS_2NDEXEC,
57 EVMCS_3RDEXEC,
58 EVMCS_PINCTRL,
59 EVMCS_VMFUNC,
60 NR_EVMCS_CTRLS,
61};
62
63static const u32 evmcs_supported_ctrls[NR_EVMCS_CTRLS][NR_EVMCS_REVISIONS] = {
64 [EVMCS_EXIT_CTRLS] = {
65 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMEXIT_CTRL,
66 },
67 [EVMCS_ENTRY_CTRLS] = {
68 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMENTRY_CTRL,
69 },
70 [EVMCS_EXEC_CTRL] = {
71 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_EXEC_CTRL,
72 },
73 [EVMCS_2NDEXEC] = {
74 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_2NDEXEC & ~SECONDARY_EXEC_TSC_SCALING,
75 },
76 [EVMCS_3RDEXEC] = {
77 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_3RDEXEC,
78 },
79 [EVMCS_PINCTRL] = {
80 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_PINCTRL,
81 },
82 [EVMCS_VMFUNC] = {
83 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMFUNC,
84 },
85};
86
87static u32 evmcs_get_supported_ctls(enum evmcs_ctrl_type ctrl_type)
88{
89 enum evmcs_revision evmcs_rev = EVMCSv1_LEGACY;
90
91 return evmcs_supported_ctrls[ctrl_type][evmcs_rev];
92}
93
94static bool evmcs_has_perf_global_ctrl(struct kvm_vcpu *vcpu)
95{
96 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
97
98 /*
99 * PERF_GLOBAL_CTRL has a quirk where some Windows guests may fail to
100 * boot if a PV CPUID feature flag is not also set. Treat the fields
101 * as unsupported if the flag is not set in guest CPUID. This should
102 * be called only for guest accesses, and all guest accesses should be
103 * gated on Hyper-V being enabled and initialized.
104 */
105 if (WARN_ON_ONCE(!hv_vcpu))
106 return false;
107
108 return hv_vcpu->cpuid_cache.nested_ebx & HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL;
109}
110
111void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
112{
113 u32 ctl_low = (u32)*pdata;
114 u32 ctl_high = (u32)(*pdata >> 32);
115 u32 supported_ctrls;
116
117 /*
118 * Hyper-V 2016 and 2019 try using these features even when eVMCS
119 * is enabled but there are no corresponding fields.
120 */
121 switch (msr_index) {
122 case MSR_IA32_VMX_EXIT_CTLS:
123 case MSR_IA32_VMX_TRUE_EXIT_CTLS:
124 supported_ctrls = evmcs_get_supported_ctls(EVMCS_EXIT_CTRLS);
125 if (!evmcs_has_perf_global_ctrl(vcpu))
126 supported_ctrls &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
127 ctl_high &= supported_ctrls;
128 break;
129 case MSR_IA32_VMX_ENTRY_CTLS:
130 case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
131 supported_ctrls = evmcs_get_supported_ctls(EVMCS_ENTRY_CTRLS);
132 if (!evmcs_has_perf_global_ctrl(vcpu))
133 supported_ctrls &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
134 ctl_high &= supported_ctrls;
135 break;
136 case MSR_IA32_VMX_PROCBASED_CTLS:
137 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
138 ctl_high &= evmcs_get_supported_ctls(EVMCS_EXEC_CTRL);
139 break;
140 case MSR_IA32_VMX_PROCBASED_CTLS2:
141 ctl_high &= evmcs_get_supported_ctls(EVMCS_2NDEXEC);
142 break;
143 case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
144 case MSR_IA32_VMX_PINBASED_CTLS:
145 ctl_high &= evmcs_get_supported_ctls(EVMCS_PINCTRL);
146 break;
147 case MSR_IA32_VMX_VMFUNC:
148 ctl_low &= evmcs_get_supported_ctls(EVMCS_VMFUNC);
149 break;
150 }
151
152 *pdata = ctl_low | ((u64)ctl_high << 32);
153}
154
155static bool nested_evmcs_is_valid_controls(enum evmcs_ctrl_type ctrl_type,
156 u32 val)
157{
158 return !(val & ~evmcs_get_supported_ctls(ctrl_type));
159}
160
161int nested_evmcs_check_controls(struct vmcs12 *vmcs12)
162{
163 if (CC(!nested_evmcs_is_valid_controls(EVMCS_PINCTRL,
164 vmcs12->pin_based_vm_exec_control)))
165 return -EINVAL;
166
167 if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXEC_CTRL,
168 vmcs12->cpu_based_vm_exec_control)))
169 return -EINVAL;
170
171 if (CC(!nested_evmcs_is_valid_controls(EVMCS_2NDEXEC,
172 vmcs12->secondary_vm_exec_control)))
173 return -EINVAL;
174
175 if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXIT_CTRLS,
176 vmcs12->vm_exit_controls)))
177 return -EINVAL;
178
179 if (CC(!nested_evmcs_is_valid_controls(EVMCS_ENTRY_CTRLS,
180 vmcs12->vm_entry_controls)))
181 return -EINVAL;
182
183 /*
184 * VM-Func controls are 64-bit, but KVM currently doesn't support any
185 * controls in bits 63:32, i.e. dropping those bits on the consistency
186 * check is intentional.
187 */
188 if (WARN_ON_ONCE(vmcs12->vm_function_control >> 32))
189 return -EINVAL;
190
191 if (CC(!nested_evmcs_is_valid_controls(EVMCS_VMFUNC,
192 vmcs12->vm_function_control)))
193 return -EINVAL;
194
195 return 0;
196}
197
198int nested_enable_evmcs(struct kvm_vcpu *vcpu,
199 uint16_t *vmcs_version)
200{
201 struct vcpu_vmx *vmx = to_vmx(vcpu);
202
203 vmx->nested.enlightened_vmcs_enabled = true;
204
205 if (vmcs_version)
206 *vmcs_version = nested_get_evmcs_version(vcpu);
207
208 return 0;
209}
210
211bool nested_evmcs_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu)
212{
213 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
214 struct vcpu_vmx *vmx = to_vmx(vcpu);
215 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
216
217 if (!hv_vcpu || !evmcs)
218 return false;
219
220 if (!evmcs->hv_enlightenments_control.nested_flush_hypercall)
221 return false;
222
223 return hv_vcpu->vp_assist_page.nested_control.features.directhypercall;
224}
225
226void vmx_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu)
227{
228 nested_vmx_vmexit(vcpu, HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH, 0, 0);
229}