Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Fault injection for both 32 and 64bit guests.
4 *
5 * Copyright (C) 2012,2013 - ARM Ltd
6 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 *
8 * Based on arch/arm/kvm/emulate.c
9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
10 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
11 */
12
13#include <hyp/adjust_pc.h>
14#include <linux/kvm_host.h>
15#include <asm/kvm_emulate.h>
16#include <asm/kvm_mmu.h>
17
18#if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__)
19#error Hypervisor code only!
20#endif
21
22static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
23{
24 u64 val;
25
26 if (__vcpu_read_sys_reg_from_cpu(reg, &val))
27 return val;
28
29 return __vcpu_sys_reg(vcpu, reg);
30}
31
32static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
33{
34 if (__vcpu_write_sys_reg_to_cpu(val, reg))
35 return;
36
37 __vcpu_sys_reg(vcpu, reg) = val;
38}
39
40static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, u64 val)
41{
42 if (has_vhe())
43 write_sysreg_el1(val, SYS_SPSR);
44 else
45 __vcpu_sys_reg(vcpu, SPSR_EL1) = val;
46}
47
48static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val)
49{
50 if (has_vhe())
51 write_sysreg(val, spsr_abt);
52 else
53 vcpu->arch.ctxt.spsr_abt = val;
54}
55
56static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val)
57{
58 if (has_vhe())
59 write_sysreg(val, spsr_und);
60 else
61 vcpu->arch.ctxt.spsr_und = val;
62}
63
64/*
65 * This performs the exception entry at a given EL (@target_mode), stashing PC
66 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE.
67 * The EL passed to this function *must* be a non-secure, privileged mode with
68 * bit 0 being set (PSTATE.SP == 1).
69 *
70 * When an exception is taken, most PSTATE fields are left unchanged in the
71 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all
72 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx
73 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0.
74 *
75 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429.
76 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426.
77 *
78 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from
79 * MSB to LSB.
80 */
81static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
82 enum exception_type type)
83{
84 unsigned long sctlr, vbar, old, new, mode;
85 u64 exc_offset;
86
87 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
88
89 if (mode == target_mode)
90 exc_offset = CURRENT_EL_SP_ELx_VECTOR;
91 else if ((mode | PSR_MODE_THREAD_BIT) == target_mode)
92 exc_offset = CURRENT_EL_SP_EL0_VECTOR;
93 else if (!(mode & PSR_MODE32_BIT))
94 exc_offset = LOWER_EL_AArch64_VECTOR;
95 else
96 exc_offset = LOWER_EL_AArch32_VECTOR;
97
98 switch (target_mode) {
99 case PSR_MODE_EL1h:
100 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1);
101 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
102 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1);
103 break;
104 default:
105 /* Don't do that */
106 BUG();
107 }
108
109 *vcpu_pc(vcpu) = vbar + exc_offset + type;
110
111 old = *vcpu_cpsr(vcpu);
112 new = 0;
113
114 new |= (old & PSR_N_BIT);
115 new |= (old & PSR_Z_BIT);
116 new |= (old & PSR_C_BIT);
117 new |= (old & PSR_V_BIT);
118
119 if (kvm_has_mte(kern_hyp_va(vcpu->kvm)))
120 new |= PSR_TCO_BIT;
121
122 new |= (old & PSR_DIT_BIT);
123
124 // PSTATE.UAO is set to zero upon any exception to AArch64
125 // See ARM DDI 0487E.a, page D5-2579.
126
127 // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0
128 // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented
129 // See ARM DDI 0487E.a, page D5-2578.
130 new |= (old & PSR_PAN_BIT);
131 if (!(sctlr & SCTLR_EL1_SPAN))
132 new |= PSR_PAN_BIT;
133
134 // PSTATE.SS is set to zero upon any exception to AArch64
135 // See ARM DDI 0487E.a, page D2-2452.
136
137 // PSTATE.IL is set to zero upon any exception to AArch64
138 // See ARM DDI 0487E.a, page D1-2306.
139
140 // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64
141 // See ARM DDI 0487E.a, page D13-3258
142 if (sctlr & SCTLR_ELx_DSSBS)
143 new |= PSR_SSBS_BIT;
144
145 // PSTATE.BTYPE is set to zero upon any exception to AArch64
146 // See ARM DDI 0487E.a, pages D1-2293 to D1-2294.
147
148 new |= PSR_D_BIT;
149 new |= PSR_A_BIT;
150 new |= PSR_I_BIT;
151 new |= PSR_F_BIT;
152
153 new |= target_mode;
154
155 *vcpu_cpsr(vcpu) = new;
156 __vcpu_write_spsr(vcpu, old);
157}
158
159/*
160 * When an exception is taken, most CPSR fields are left unchanged in the
161 * handler. However, some are explicitly overridden (e.g. M[4:0]).
162 *
163 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with
164 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was
165 * obsoleted by the ARMv7 virtualization extensions and is RES0.
166 *
167 * For the SPSR layout seen from AArch32, see:
168 * - ARM DDI 0406C.d, page B1-1148
169 * - ARM DDI 0487E.a, page G8-6264
170 *
171 * For the SPSR_ELx layout for AArch32 seen from AArch64, see:
172 * - ARM DDI 0487E.a, page C5-426
173 *
174 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from
175 * MSB to LSB.
176 */
177static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode)
178{
179 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
180 unsigned long old, new;
181
182 old = *vcpu_cpsr(vcpu);
183 new = 0;
184
185 new |= (old & PSR_AA32_N_BIT);
186 new |= (old & PSR_AA32_Z_BIT);
187 new |= (old & PSR_AA32_C_BIT);
188 new |= (old & PSR_AA32_V_BIT);
189 new |= (old & PSR_AA32_Q_BIT);
190
191 // CPSR.IT[7:0] are set to zero upon any exception
192 // See ARM DDI 0487E.a, section G1.12.3
193 // See ARM DDI 0406C.d, section B1.8.3
194
195 new |= (old & PSR_AA32_DIT_BIT);
196
197 // CPSR.SSBS is set to SCTLR.DSSBS upon any exception
198 // See ARM DDI 0487E.a, page G8-6244
199 if (sctlr & BIT(31))
200 new |= PSR_AA32_SSBS_BIT;
201
202 // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0
203 // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented
204 // See ARM DDI 0487E.a, page G8-6246
205 new |= (old & PSR_AA32_PAN_BIT);
206 if (!(sctlr & BIT(23)))
207 new |= PSR_AA32_PAN_BIT;
208
209 // SS does not exist in AArch32, so ignore
210
211 // CPSR.IL is set to zero upon any exception
212 // See ARM DDI 0487E.a, page G1-5527
213
214 new |= (old & PSR_AA32_GE_MASK);
215
216 // CPSR.IT[7:0] are set to zero upon any exception
217 // See prior comment above
218
219 // CPSR.E is set to SCTLR.EE upon any exception
220 // See ARM DDI 0487E.a, page G8-6245
221 // See ARM DDI 0406C.d, page B4-1701
222 if (sctlr & BIT(25))
223 new |= PSR_AA32_E_BIT;
224
225 // CPSR.A is unchanged upon an exception to Undefined, Supervisor
226 // CPSR.A is set upon an exception to other modes
227 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
228 // See ARM DDI 0406C.d, page B1-1182
229 new |= (old & PSR_AA32_A_BIT);
230 if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC)
231 new |= PSR_AA32_A_BIT;
232
233 // CPSR.I is set upon any exception
234 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
235 // See ARM DDI 0406C.d, page B1-1182
236 new |= PSR_AA32_I_BIT;
237
238 // CPSR.F is set upon an exception to FIQ
239 // CPSR.F is unchanged upon an exception to other modes
240 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
241 // See ARM DDI 0406C.d, page B1-1182
242 new |= (old & PSR_AA32_F_BIT);
243 if (mode == PSR_AA32_MODE_FIQ)
244 new |= PSR_AA32_F_BIT;
245
246 // CPSR.T is set to SCTLR.TE upon any exception
247 // See ARM DDI 0487E.a, page G8-5514
248 // See ARM DDI 0406C.d, page B1-1181
249 if (sctlr & BIT(30))
250 new |= PSR_AA32_T_BIT;
251
252 new |= mode;
253
254 return new;
255}
256
257/*
258 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10.
259 */
260static const u8 return_offsets[8][2] = {
261 [0] = { 0, 0 }, /* Reset, unused */
262 [1] = { 4, 2 }, /* Undefined */
263 [2] = { 0, 0 }, /* SVC, unused */
264 [3] = { 4, 4 }, /* Prefetch abort */
265 [4] = { 8, 8 }, /* Data abort */
266 [5] = { 0, 0 }, /* HVC, unused */
267 [6] = { 4, 4 }, /* IRQ, unused */
268 [7] = { 4, 4 }, /* FIQ, unused */
269};
270
271static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
272{
273 unsigned long spsr = *vcpu_cpsr(vcpu);
274 bool is_thumb = (spsr & PSR_AA32_T_BIT);
275 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
276 u32 return_address;
277
278 *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode);
279 return_address = *vcpu_pc(vcpu);
280 return_address += return_offsets[vect_offset >> 2][is_thumb];
281
282 /* KVM only enters the ABT and UND modes, so only deal with those */
283 switch(mode) {
284 case PSR_AA32_MODE_ABT:
285 __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr));
286 vcpu_gp_regs(vcpu)->compat_lr_abt = return_address;
287 break;
288
289 case PSR_AA32_MODE_UND:
290 __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr));
291 vcpu_gp_regs(vcpu)->compat_lr_und = return_address;
292 break;
293 }
294
295 /* Branch to exception vector */
296 if (sctlr & (1 << 13))
297 vect_offset += 0xffff0000;
298 else /* always have security exceptions */
299 vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1);
300
301 *vcpu_pc(vcpu) = vect_offset;
302}
303
304static void kvm_inject_exception(struct kvm_vcpu *vcpu)
305{
306 if (vcpu_el1_is_32bit(vcpu)) {
307 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
308 case unpack_vcpu_flag(EXCEPT_AA32_UND):
309 enter_exception32(vcpu, PSR_AA32_MODE_UND, 4);
310 break;
311 case unpack_vcpu_flag(EXCEPT_AA32_IABT):
312 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12);
313 break;
314 case unpack_vcpu_flag(EXCEPT_AA32_DABT):
315 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16);
316 break;
317 default:
318 /* Err... */
319 break;
320 }
321 } else {
322 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
323 case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):
324 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
325 break;
326 default:
327 /*
328 * Only EL1_SYNC makes sense so far, EL2_{SYNC,IRQ}
329 * will be implemented at some point. Everything
330 * else gets silently ignored.
331 */
332 break;
333 }
334 }
335}
336
337/*
338 * Adjust the guest PC (and potentially exception state) depending on
339 * flags provided by the emulation code.
340 */
341void __kvm_adjust_pc(struct kvm_vcpu *vcpu)
342{
343 if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) {
344 kvm_inject_exception(vcpu);
345 vcpu_clear_flag(vcpu, PENDING_EXCEPTION);
346 vcpu_clear_flag(vcpu, EXCEPT_MASK);
347 } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) {
348 kvm_skip_instr(vcpu);
349 vcpu_clear_flag(vcpu, INCREMENT_PC);
350 }
351}