Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6#include <linux/jump_label.h>
7
8#include <asm/kvm_asm.h>
9#include <asm/kvm_hyp.h>
10#include <asm/kvm_mmu.h>
11
12__asm__(".arch_extension virt");
13
14/*
15 * Activate the traps, saving the host's fpexc register before
16 * overwriting it. We'll restore it on VM exit.
17 */
18static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu, u32 *fpexc_host)
19{
20 u32 val;
21
22 /*
23 * We are about to set HCPTR.TCP10/11 to trap all floating point
24 * register accesses to HYP, however, the ARM ARM clearly states that
25 * traps are only taken to HYP if the operation would not otherwise
26 * trap to SVC. Therefore, always make sure that for 32-bit guests,
27 * we set FPEXC.EN to prevent traps to SVC, when setting the TCP bits.
28 */
29 val = read_sysreg(VFP_FPEXC);
30 *fpexc_host = val;
31 if (!(val & FPEXC_EN)) {
32 write_sysreg(val | FPEXC_EN, VFP_FPEXC);
33 isb();
34 }
35
36 write_sysreg(vcpu->arch.hcr, HCR);
37 /* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
38 write_sysreg(HSTR_T(15), HSTR);
39 write_sysreg(HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11), HCPTR);
40 val = read_sysreg(HDCR);
41 val |= HDCR_TPM | HDCR_TPMCR; /* trap performance monitors */
42 val |= HDCR_TDRA | HDCR_TDOSA | HDCR_TDA; /* trap debug regs */
43 write_sysreg(val, HDCR);
44}
45
46static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
47{
48 u32 val;
49
50 /*
51 * If we pended a virtual abort, preserve it until it gets
52 * cleared. See B1.9.9 (Virtual Abort exception) for details,
53 * but the crucial bit is the zeroing of HCR.VA in the
54 * pseudocode.
55 */
56 if (vcpu->arch.hcr & HCR_VA)
57 vcpu->arch.hcr = read_sysreg(HCR);
58
59 write_sysreg(0, HCR);
60 write_sysreg(0, HSTR);
61 val = read_sysreg(HDCR);
62 write_sysreg(val & ~(HDCR_TPM | HDCR_TPMCR), HDCR);
63 write_sysreg(0, HCPTR);
64}
65
66static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
67{
68 struct kvm *kvm = kern_hyp_va(vcpu->kvm);
69 write_sysreg(kvm_get_vttbr(kvm), VTTBR);
70 write_sysreg(vcpu->arch.midr, VPIDR);
71}
72
73static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
74{
75 write_sysreg(0, VTTBR);
76 write_sysreg(read_sysreg(MIDR), VPIDR);
77}
78
79
80static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
81{
82 if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
83 __vgic_v3_save_state(vcpu);
84 __vgic_v3_deactivate_traps(vcpu);
85 }
86}
87
88static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
89{
90 if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
91 __vgic_v3_activate_traps(vcpu);
92 __vgic_v3_restore_state(vcpu);
93 }
94}
95
96static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
97{
98 u32 hsr = read_sysreg(HSR);
99 u8 ec = hsr >> HSR_EC_SHIFT;
100 u32 hpfar, far;
101
102 vcpu->arch.fault.hsr = hsr;
103
104 if (ec == HSR_EC_IABT)
105 far = read_sysreg(HIFAR);
106 else if (ec == HSR_EC_DABT)
107 far = read_sysreg(HDFAR);
108 else
109 return true;
110
111 /*
112 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
113 *
114 * Abort on the stage 2 translation for a memory access from a
115 * Non-secure PL1 or PL0 mode:
116 *
117 * For any Access flag fault or Translation fault, and also for any
118 * Permission fault on the stage 2 translation of a memory access
119 * made as part of a translation table walk for a stage 1 translation,
120 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
121 * is UNKNOWN.
122 */
123 if (!(hsr & HSR_DABT_S1PTW) && (hsr & HSR_FSC_TYPE) == FSC_PERM) {
124 u64 par, tmp;
125
126 par = read_sysreg(PAR);
127 write_sysreg(far, ATS1CPR);
128 isb();
129
130 tmp = read_sysreg(PAR);
131 write_sysreg(par, PAR);
132
133 if (unlikely(tmp & 1))
134 return false; /* Translation failed, back to guest */
135
136 hpfar = ((tmp >> 12) & ((1UL << 28) - 1)) << 4;
137 } else {
138 hpfar = read_sysreg(HPFAR);
139 }
140
141 vcpu->arch.fault.hxfar = far;
142 vcpu->arch.fault.hpfar = hpfar;
143 return true;
144}
145
146int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
147{
148 struct kvm_cpu_context *host_ctxt;
149 struct kvm_cpu_context *guest_ctxt;
150 bool fp_enabled;
151 u64 exit_code;
152 u32 fpexc;
153
154 vcpu = kern_hyp_va(vcpu);
155 write_sysreg(vcpu, HTPIDR);
156
157 host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
158 guest_ctxt = &vcpu->arch.ctxt;
159
160 __sysreg_save_state(host_ctxt);
161 __banked_save_state(host_ctxt);
162
163 __activate_traps(vcpu, &fpexc);
164 __activate_vm(vcpu);
165
166 __vgic_restore_state(vcpu);
167 __timer_enable_traps(vcpu);
168
169 __sysreg_restore_state(guest_ctxt);
170 __banked_restore_state(guest_ctxt);
171
172 /* Jump in the fire! */
173again:
174 exit_code = __guest_enter(vcpu, host_ctxt);
175 /* And we're baaack! */
176
177 if (exit_code == ARM_EXCEPTION_HVC && !__populate_fault_info(vcpu))
178 goto again;
179
180 fp_enabled = __vfp_enabled();
181
182 __banked_save_state(guest_ctxt);
183 __sysreg_save_state(guest_ctxt);
184 __timer_disable_traps(vcpu);
185
186 __vgic_save_state(vcpu);
187
188 __deactivate_traps(vcpu);
189 __deactivate_vm(vcpu);
190
191 __banked_restore_state(host_ctxt);
192 __sysreg_restore_state(host_ctxt);
193
194 if (fp_enabled) {
195 __vfp_save_state(&guest_ctxt->vfp);
196 __vfp_restore_state(&host_ctxt->vfp);
197 }
198
199 write_sysreg(fpexc, VFP_FPEXC);
200
201 return exit_code;
202}
203
204static const char * const __hyp_panic_string[] = {
205 [ARM_EXCEPTION_RESET] = "\nHYP panic: RST PC:%08x CPSR:%08x",
206 [ARM_EXCEPTION_UNDEFINED] = "\nHYP panic: UNDEF PC:%08x CPSR:%08x",
207 [ARM_EXCEPTION_SOFTWARE] = "\nHYP panic: SVC PC:%08x CPSR:%08x",
208 [ARM_EXCEPTION_PREF_ABORT] = "\nHYP panic: PABRT PC:%08x CPSR:%08x",
209 [ARM_EXCEPTION_DATA_ABORT] = "\nHYP panic: DABRT PC:%08x ADDR:%08x",
210 [ARM_EXCEPTION_IRQ] = "\nHYP panic: IRQ PC:%08x CPSR:%08x",
211 [ARM_EXCEPTION_FIQ] = "\nHYP panic: FIQ PC:%08x CPSR:%08x",
212 [ARM_EXCEPTION_HVC] = "\nHYP panic: HVC PC:%08x CPSR:%08x",
213};
214
215void __hyp_text __noreturn __hyp_panic(int cause)
216{
217 u32 elr = read_special(ELR_hyp);
218 u32 val;
219
220 if (cause == ARM_EXCEPTION_DATA_ABORT)
221 val = read_sysreg(HDFAR);
222 else
223 val = read_special(SPSR);
224
225 if (read_sysreg(VTTBR)) {
226 struct kvm_vcpu *vcpu;
227 struct kvm_cpu_context *host_ctxt;
228
229 vcpu = (struct kvm_vcpu *)read_sysreg(HTPIDR);
230 host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
231 __timer_disable_traps(vcpu);
232 __deactivate_traps(vcpu);
233 __deactivate_vm(vcpu);
234 __banked_restore_state(host_ctxt);
235 __sysreg_restore_state(host_ctxt);
236 }
237
238 /* Call panic for real */
239 __hyp_do_panic(__hyp_panic_string[cause], elr, val);
240
241 unreachable();
242}
1/*
2 * Copyright (C) 2015 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17#include <linux/jump_label.h>
18
19#include <asm/kvm_asm.h>
20#include <asm/kvm_hyp.h>
21
22__asm__(".arch_extension virt");
23
24/*
25 * Activate the traps, saving the host's fpexc register before
26 * overwriting it. We'll restore it on VM exit.
27 */
28static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu, u32 *fpexc_host)
29{
30 u32 val;
31
32 /*
33 * We are about to set HCPTR.TCP10/11 to trap all floating point
34 * register accesses to HYP, however, the ARM ARM clearly states that
35 * traps are only taken to HYP if the operation would not otherwise
36 * trap to SVC. Therefore, always make sure that for 32-bit guests,
37 * we set FPEXC.EN to prevent traps to SVC, when setting the TCP bits.
38 */
39 val = read_sysreg(VFP_FPEXC);
40 *fpexc_host = val;
41 if (!(val & FPEXC_EN)) {
42 write_sysreg(val | FPEXC_EN, VFP_FPEXC);
43 isb();
44 }
45
46 write_sysreg(vcpu->arch.hcr | vcpu->arch.irq_lines, HCR);
47 /* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
48 write_sysreg(HSTR_T(15), HSTR);
49 write_sysreg(HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11), HCPTR);
50 val = read_sysreg(HDCR);
51 write_sysreg(val | HDCR_TPM | HDCR_TPMCR, HDCR);
52}
53
54static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
55{
56 u32 val;
57
58 /*
59 * If we pended a virtual abort, preserve it until it gets
60 * cleared. See B1.9.9 (Virtual Abort exception) for details,
61 * but the crucial bit is the zeroing of HCR.VA in the
62 * pseudocode.
63 */
64 if (vcpu->arch.hcr & HCR_VA)
65 vcpu->arch.hcr = read_sysreg(HCR);
66
67 write_sysreg(0, HCR);
68 write_sysreg(0, HSTR);
69 val = read_sysreg(HDCR);
70 write_sysreg(val & ~(HDCR_TPM | HDCR_TPMCR), HDCR);
71 write_sysreg(0, HCPTR);
72}
73
74static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
75{
76 struct kvm *kvm = kern_hyp_va(vcpu->kvm);
77 write_sysreg(kvm->arch.vttbr, VTTBR);
78 write_sysreg(vcpu->arch.midr, VPIDR);
79}
80
81static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
82{
83 write_sysreg(0, VTTBR);
84 write_sysreg(read_sysreg(MIDR), VPIDR);
85}
86
87
88static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
89{
90 if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
91 __vgic_v3_save_state(vcpu);
92 else
93 __vgic_v2_save_state(vcpu);
94}
95
96static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
97{
98 if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
99 __vgic_v3_restore_state(vcpu);
100 else
101 __vgic_v2_restore_state(vcpu);
102}
103
104static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
105{
106 u32 hsr = read_sysreg(HSR);
107 u8 ec = hsr >> HSR_EC_SHIFT;
108 u32 hpfar, far;
109
110 vcpu->arch.fault.hsr = hsr;
111
112 if (ec == HSR_EC_IABT)
113 far = read_sysreg(HIFAR);
114 else if (ec == HSR_EC_DABT)
115 far = read_sysreg(HDFAR);
116 else
117 return true;
118
119 /*
120 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
121 *
122 * Abort on the stage 2 translation for a memory access from a
123 * Non-secure PL1 or PL0 mode:
124 *
125 * For any Access flag fault or Translation fault, and also for any
126 * Permission fault on the stage 2 translation of a memory access
127 * made as part of a translation table walk for a stage 1 translation,
128 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
129 * is UNKNOWN.
130 */
131 if (!(hsr & HSR_DABT_S1PTW) && (hsr & HSR_FSC_TYPE) == FSC_PERM) {
132 u64 par, tmp;
133
134 par = read_sysreg(PAR);
135 write_sysreg(far, ATS1CPR);
136 isb();
137
138 tmp = read_sysreg(PAR);
139 write_sysreg(par, PAR);
140
141 if (unlikely(tmp & 1))
142 return false; /* Translation failed, back to guest */
143
144 hpfar = ((tmp >> 12) & ((1UL << 28) - 1)) << 4;
145 } else {
146 hpfar = read_sysreg(HPFAR);
147 }
148
149 vcpu->arch.fault.hxfar = far;
150 vcpu->arch.fault.hpfar = hpfar;
151 return true;
152}
153
154int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
155{
156 struct kvm_cpu_context *host_ctxt;
157 struct kvm_cpu_context *guest_ctxt;
158 bool fp_enabled;
159 u64 exit_code;
160 u32 fpexc;
161
162 vcpu = kern_hyp_va(vcpu);
163 write_sysreg(vcpu, HTPIDR);
164
165 host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
166 guest_ctxt = &vcpu->arch.ctxt;
167
168 __sysreg_save_state(host_ctxt);
169 __banked_save_state(host_ctxt);
170
171 __activate_traps(vcpu, &fpexc);
172 __activate_vm(vcpu);
173
174 __vgic_restore_state(vcpu);
175 __timer_restore_state(vcpu);
176
177 __sysreg_restore_state(guest_ctxt);
178 __banked_restore_state(guest_ctxt);
179
180 /* Jump in the fire! */
181again:
182 exit_code = __guest_enter(vcpu, host_ctxt);
183 /* And we're baaack! */
184
185 if (exit_code == ARM_EXCEPTION_HVC && !__populate_fault_info(vcpu))
186 goto again;
187
188 fp_enabled = __vfp_enabled();
189
190 __banked_save_state(guest_ctxt);
191 __sysreg_save_state(guest_ctxt);
192 __timer_save_state(vcpu);
193 __vgic_save_state(vcpu);
194
195 __deactivate_traps(vcpu);
196 __deactivate_vm(vcpu);
197
198 __banked_restore_state(host_ctxt);
199 __sysreg_restore_state(host_ctxt);
200
201 if (fp_enabled) {
202 __vfp_save_state(&guest_ctxt->vfp);
203 __vfp_restore_state(&host_ctxt->vfp);
204 }
205
206 write_sysreg(fpexc, VFP_FPEXC);
207
208 return exit_code;
209}
210
211static const char * const __hyp_panic_string[] = {
212 [ARM_EXCEPTION_RESET] = "\nHYP panic: RST PC:%08x CPSR:%08x",
213 [ARM_EXCEPTION_UNDEFINED] = "\nHYP panic: UNDEF PC:%08x CPSR:%08x",
214 [ARM_EXCEPTION_SOFTWARE] = "\nHYP panic: SVC PC:%08x CPSR:%08x",
215 [ARM_EXCEPTION_PREF_ABORT] = "\nHYP panic: PABRT PC:%08x CPSR:%08x",
216 [ARM_EXCEPTION_DATA_ABORT] = "\nHYP panic: DABRT PC:%08x ADDR:%08x",
217 [ARM_EXCEPTION_IRQ] = "\nHYP panic: IRQ PC:%08x CPSR:%08x",
218 [ARM_EXCEPTION_FIQ] = "\nHYP panic: FIQ PC:%08x CPSR:%08x",
219 [ARM_EXCEPTION_HVC] = "\nHYP panic: HVC PC:%08x CPSR:%08x",
220};
221
222void __hyp_text __noreturn __hyp_panic(int cause)
223{
224 u32 elr = read_special(ELR_hyp);
225 u32 val;
226
227 if (cause == ARM_EXCEPTION_DATA_ABORT)
228 val = read_sysreg(HDFAR);
229 else
230 val = read_special(SPSR);
231
232 if (read_sysreg(VTTBR)) {
233 struct kvm_vcpu *vcpu;
234 struct kvm_cpu_context *host_ctxt;
235
236 vcpu = (struct kvm_vcpu *)read_sysreg(HTPIDR);
237 host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
238 __deactivate_traps(vcpu);
239 __deactivate_vm(vcpu);
240 __sysreg_restore_state(host_ctxt);
241 }
242
243 /* Call panic for real */
244 __hyp_do_panic(__hyp_panic_string[cause], elr, val);
245
246 unreachable();
247}