Loading...
1/*
2 * Copyright (C) 2012,2013 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * Derived from arch/arm/kvm/coproc.c:
6 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
7 * Authors: Rusty Russell <rusty@rustcorp.com.au>
8 * Christoffer Dall <c.dall@virtualopensystems.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License, version 2, as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 */
22
23#include <linux/bsearch.h>
24#include <linux/kvm_host.h>
25#include <linux/mm.h>
26#include <linux/printk.h>
27#include <linux/uaccess.h>
28
29#include <asm/cacheflush.h>
30#include <asm/cputype.h>
31#include <asm/debug-monitors.h>
32#include <asm/esr.h>
33#include <asm/kvm_arm.h>
34#include <asm/kvm_asm.h>
35#include <asm/kvm_coproc.h>
36#include <asm/kvm_emulate.h>
37#include <asm/kvm_host.h>
38#include <asm/kvm_hyp.h>
39#include <asm/kvm_mmu.h>
40#include <asm/perf_event.h>
41#include <asm/sysreg.h>
42
43#include <trace/events/kvm.h>
44
45#include "sys_regs.h"
46
47#include "trace.h"
48
49/*
50 * All of this file is extremly similar to the ARM coproc.c, but the
51 * types are different. My gut feeling is that it should be pretty
52 * easy to merge, but that would be an ABI breakage -- again. VFP
53 * would also need to be abstracted.
54 *
55 * For AArch32, we only take care of what is being trapped. Anything
56 * that has to do with init and userspace access has to go via the
57 * 64bit interface.
58 */
59
60static bool read_from_write_only(struct kvm_vcpu *vcpu,
61 struct sys_reg_params *params,
62 const struct sys_reg_desc *r)
63{
64 WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
65 print_sys_reg_instr(params);
66 kvm_inject_undefined(vcpu);
67 return false;
68}
69
70static bool write_to_read_only(struct kvm_vcpu *vcpu,
71 struct sys_reg_params *params,
72 const struct sys_reg_desc *r)
73{
74 WARN_ONCE(1, "Unexpected sys_reg write to read-only register\n");
75 print_sys_reg_instr(params);
76 kvm_inject_undefined(vcpu);
77 return false;
78}
79
80u64 vcpu_read_sys_reg(struct kvm_vcpu *vcpu, int reg)
81{
82 if (!vcpu->arch.sysregs_loaded_on_cpu)
83 goto immediate_read;
84
85 /*
86 * System registers listed in the switch are not saved on every
87 * exit from the guest but are only saved on vcpu_put.
88 *
89 * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
90 * should never be listed below, because the guest cannot modify its
91 * own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
92 * thread when emulating cross-VCPU communication.
93 */
94 switch (reg) {
95 case CSSELR_EL1: return read_sysreg_s(SYS_CSSELR_EL1);
96 case SCTLR_EL1: return read_sysreg_s(sctlr_EL12);
97 case ACTLR_EL1: return read_sysreg_s(SYS_ACTLR_EL1);
98 case CPACR_EL1: return read_sysreg_s(cpacr_EL12);
99 case TTBR0_EL1: return read_sysreg_s(ttbr0_EL12);
100 case TTBR1_EL1: return read_sysreg_s(ttbr1_EL12);
101 case TCR_EL1: return read_sysreg_s(tcr_EL12);
102 case ESR_EL1: return read_sysreg_s(esr_EL12);
103 case AFSR0_EL1: return read_sysreg_s(afsr0_EL12);
104 case AFSR1_EL1: return read_sysreg_s(afsr1_EL12);
105 case FAR_EL1: return read_sysreg_s(far_EL12);
106 case MAIR_EL1: return read_sysreg_s(mair_EL12);
107 case VBAR_EL1: return read_sysreg_s(vbar_EL12);
108 case CONTEXTIDR_EL1: return read_sysreg_s(contextidr_EL12);
109 case TPIDR_EL0: return read_sysreg_s(SYS_TPIDR_EL0);
110 case TPIDRRO_EL0: return read_sysreg_s(SYS_TPIDRRO_EL0);
111 case TPIDR_EL1: return read_sysreg_s(SYS_TPIDR_EL1);
112 case AMAIR_EL1: return read_sysreg_s(amair_EL12);
113 case CNTKCTL_EL1: return read_sysreg_s(cntkctl_EL12);
114 case PAR_EL1: return read_sysreg_s(SYS_PAR_EL1);
115 case DACR32_EL2: return read_sysreg_s(SYS_DACR32_EL2);
116 case IFSR32_EL2: return read_sysreg_s(SYS_IFSR32_EL2);
117 case DBGVCR32_EL2: return read_sysreg_s(SYS_DBGVCR32_EL2);
118 }
119
120immediate_read:
121 return __vcpu_sys_reg(vcpu, reg);
122}
123
124void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
125{
126 if (!vcpu->arch.sysregs_loaded_on_cpu)
127 goto immediate_write;
128
129 /*
130 * System registers listed in the switch are not restored on every
131 * entry to the guest but are only restored on vcpu_load.
132 *
133 * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
134 * should never be listed below, because the the MPIDR should only be
135 * set once, before running the VCPU, and never changed later.
136 */
137 switch (reg) {
138 case CSSELR_EL1: write_sysreg_s(val, SYS_CSSELR_EL1); return;
139 case SCTLR_EL1: write_sysreg_s(val, sctlr_EL12); return;
140 case ACTLR_EL1: write_sysreg_s(val, SYS_ACTLR_EL1); return;
141 case CPACR_EL1: write_sysreg_s(val, cpacr_EL12); return;
142 case TTBR0_EL1: write_sysreg_s(val, ttbr0_EL12); return;
143 case TTBR1_EL1: write_sysreg_s(val, ttbr1_EL12); return;
144 case TCR_EL1: write_sysreg_s(val, tcr_EL12); return;
145 case ESR_EL1: write_sysreg_s(val, esr_EL12); return;
146 case AFSR0_EL1: write_sysreg_s(val, afsr0_EL12); return;
147 case AFSR1_EL1: write_sysreg_s(val, afsr1_EL12); return;
148 case FAR_EL1: write_sysreg_s(val, far_EL12); return;
149 case MAIR_EL1: write_sysreg_s(val, mair_EL12); return;
150 case VBAR_EL1: write_sysreg_s(val, vbar_EL12); return;
151 case CONTEXTIDR_EL1: write_sysreg_s(val, contextidr_EL12); return;
152 case TPIDR_EL0: write_sysreg_s(val, SYS_TPIDR_EL0); return;
153 case TPIDRRO_EL0: write_sysreg_s(val, SYS_TPIDRRO_EL0); return;
154 case TPIDR_EL1: write_sysreg_s(val, SYS_TPIDR_EL1); return;
155 case AMAIR_EL1: write_sysreg_s(val, amair_EL12); return;
156 case CNTKCTL_EL1: write_sysreg_s(val, cntkctl_EL12); return;
157 case PAR_EL1: write_sysreg_s(val, SYS_PAR_EL1); return;
158 case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); return;
159 case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); return;
160 case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); return;
161 }
162
163immediate_write:
164 __vcpu_sys_reg(vcpu, reg) = val;
165}
166
167/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
168static u32 cache_levels;
169
170/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
171#define CSSELR_MAX 12
172
173/* Which cache CCSIDR represents depends on CSSELR value. */
174static u32 get_ccsidr(u32 csselr)
175{
176 u32 ccsidr;
177
178 /* Make sure noone else changes CSSELR during this! */
179 local_irq_disable();
180 write_sysreg(csselr, csselr_el1);
181 isb();
182 ccsidr = read_sysreg(ccsidr_el1);
183 local_irq_enable();
184
185 return ccsidr;
186}
187
188/*
189 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
190 */
191static bool access_dcsw(struct kvm_vcpu *vcpu,
192 struct sys_reg_params *p,
193 const struct sys_reg_desc *r)
194{
195 if (!p->is_write)
196 return read_from_write_only(vcpu, p, r);
197
198 kvm_set_way_flush(vcpu);
199 return true;
200}
201
202/*
203 * Generic accessor for VM registers. Only called as long as HCR_TVM
204 * is set. If the guest enables the MMU, we stop trapping the VM
205 * sys_regs and leave it in complete control of the caches.
206 */
207static bool access_vm_reg(struct kvm_vcpu *vcpu,
208 struct sys_reg_params *p,
209 const struct sys_reg_desc *r)
210{
211 bool was_enabled = vcpu_has_cache_enabled(vcpu);
212 u64 val;
213 int reg = r->reg;
214
215 BUG_ON(!p->is_write);
216
217 /* See the 32bit mapping in kvm_host.h */
218 if (p->is_aarch32)
219 reg = r->reg / 2;
220
221 if (!p->is_aarch32 || !p->is_32bit) {
222 val = p->regval;
223 } else {
224 val = vcpu_read_sys_reg(vcpu, reg);
225 if (r->reg % 2)
226 val = (p->regval << 32) | (u64)lower_32_bits(val);
227 else
228 val = ((u64)upper_32_bits(val) << 32) |
229 lower_32_bits(p->regval);
230 }
231 vcpu_write_sys_reg(vcpu, val, reg);
232
233 kvm_toggle_cache(vcpu, was_enabled);
234 return true;
235}
236
237/*
238 * Trap handler for the GICv3 SGI generation system register.
239 * Forward the request to the VGIC emulation.
240 * The cp15_64 code makes sure this automatically works
241 * for both AArch64 and AArch32 accesses.
242 */
243static bool access_gic_sgi(struct kvm_vcpu *vcpu,
244 struct sys_reg_params *p,
245 const struct sys_reg_desc *r)
246{
247 if (!p->is_write)
248 return read_from_write_only(vcpu, p, r);
249
250 vgic_v3_dispatch_sgi(vcpu, p->regval);
251
252 return true;
253}
254
255static bool access_gic_sre(struct kvm_vcpu *vcpu,
256 struct sys_reg_params *p,
257 const struct sys_reg_desc *r)
258{
259 if (p->is_write)
260 return ignore_write(vcpu, p);
261
262 p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
263 return true;
264}
265
266static bool trap_raz_wi(struct kvm_vcpu *vcpu,
267 struct sys_reg_params *p,
268 const struct sys_reg_desc *r)
269{
270 if (p->is_write)
271 return ignore_write(vcpu, p);
272 else
273 return read_zero(vcpu, p);
274}
275
276static bool trap_undef(struct kvm_vcpu *vcpu,
277 struct sys_reg_params *p,
278 const struct sys_reg_desc *r)
279{
280 kvm_inject_undefined(vcpu);
281 return false;
282}
283
284static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
285 struct sys_reg_params *p,
286 const struct sys_reg_desc *r)
287{
288 if (p->is_write) {
289 return ignore_write(vcpu, p);
290 } else {
291 p->regval = (1 << 3);
292 return true;
293 }
294}
295
296static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
297 struct sys_reg_params *p,
298 const struct sys_reg_desc *r)
299{
300 if (p->is_write) {
301 return ignore_write(vcpu, p);
302 } else {
303 p->regval = read_sysreg(dbgauthstatus_el1);
304 return true;
305 }
306}
307
308/*
309 * We want to avoid world-switching all the DBG registers all the
310 * time:
311 *
312 * - If we've touched any debug register, it is likely that we're
313 * going to touch more of them. It then makes sense to disable the
314 * traps and start doing the save/restore dance
315 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
316 * then mandatory to save/restore the registers, as the guest
317 * depends on them.
318 *
319 * For this, we use a DIRTY bit, indicating the guest has modified the
320 * debug registers, used as follow:
321 *
322 * On guest entry:
323 * - If the dirty bit is set (because we're coming back from trapping),
324 * disable the traps, save host registers, restore guest registers.
325 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
326 * set the dirty bit, disable the traps, save host registers,
327 * restore guest registers.
328 * - Otherwise, enable the traps
329 *
330 * On guest exit:
331 * - If the dirty bit is set, save guest registers, restore host
332 * registers and clear the dirty bit. This ensure that the host can
333 * now use the debug registers.
334 */
335static bool trap_debug_regs(struct kvm_vcpu *vcpu,
336 struct sys_reg_params *p,
337 const struct sys_reg_desc *r)
338{
339 if (p->is_write) {
340 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
341 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
342 } else {
343 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
344 }
345
346 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
347
348 return true;
349}
350
351/*
352 * reg_to_dbg/dbg_to_reg
353 *
354 * A 32 bit write to a debug register leave top bits alone
355 * A 32 bit read from a debug register only returns the bottom bits
356 *
357 * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the
358 * hyp.S code switches between host and guest values in future.
359 */
360static void reg_to_dbg(struct kvm_vcpu *vcpu,
361 struct sys_reg_params *p,
362 u64 *dbg_reg)
363{
364 u64 val = p->regval;
365
366 if (p->is_32bit) {
367 val &= 0xffffffffUL;
368 val |= ((*dbg_reg >> 32) << 32);
369 }
370
371 *dbg_reg = val;
372 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
373}
374
375static void dbg_to_reg(struct kvm_vcpu *vcpu,
376 struct sys_reg_params *p,
377 u64 *dbg_reg)
378{
379 p->regval = *dbg_reg;
380 if (p->is_32bit)
381 p->regval &= 0xffffffffUL;
382}
383
384static bool trap_bvr(struct kvm_vcpu *vcpu,
385 struct sys_reg_params *p,
386 const struct sys_reg_desc *rd)
387{
388 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
389
390 if (p->is_write)
391 reg_to_dbg(vcpu, p, dbg_reg);
392 else
393 dbg_to_reg(vcpu, p, dbg_reg);
394
395 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
396
397 return true;
398}
399
400static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
401 const struct kvm_one_reg *reg, void __user *uaddr)
402{
403 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
404
405 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
406 return -EFAULT;
407 return 0;
408}
409
410static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
411 const struct kvm_one_reg *reg, void __user *uaddr)
412{
413 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
414
415 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
416 return -EFAULT;
417 return 0;
418}
419
420static void reset_bvr(struct kvm_vcpu *vcpu,
421 const struct sys_reg_desc *rd)
422{
423 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val;
424}
425
426static bool trap_bcr(struct kvm_vcpu *vcpu,
427 struct sys_reg_params *p,
428 const struct sys_reg_desc *rd)
429{
430 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
431
432 if (p->is_write)
433 reg_to_dbg(vcpu, p, dbg_reg);
434 else
435 dbg_to_reg(vcpu, p, dbg_reg);
436
437 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
438
439 return true;
440}
441
442static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
443 const struct kvm_one_reg *reg, void __user *uaddr)
444{
445 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
446
447 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
448 return -EFAULT;
449
450 return 0;
451}
452
453static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
454 const struct kvm_one_reg *reg, void __user *uaddr)
455{
456 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
457
458 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
459 return -EFAULT;
460 return 0;
461}
462
463static void reset_bcr(struct kvm_vcpu *vcpu,
464 const struct sys_reg_desc *rd)
465{
466 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val;
467}
468
469static bool trap_wvr(struct kvm_vcpu *vcpu,
470 struct sys_reg_params *p,
471 const struct sys_reg_desc *rd)
472{
473 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
474
475 if (p->is_write)
476 reg_to_dbg(vcpu, p, dbg_reg);
477 else
478 dbg_to_reg(vcpu, p, dbg_reg);
479
480 trace_trap_reg(__func__, rd->reg, p->is_write,
481 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]);
482
483 return true;
484}
485
486static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
487 const struct kvm_one_reg *reg, void __user *uaddr)
488{
489 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
490
491 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
492 return -EFAULT;
493 return 0;
494}
495
496static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
497 const struct kvm_one_reg *reg, void __user *uaddr)
498{
499 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
500
501 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
502 return -EFAULT;
503 return 0;
504}
505
506static void reset_wvr(struct kvm_vcpu *vcpu,
507 const struct sys_reg_desc *rd)
508{
509 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val;
510}
511
512static bool trap_wcr(struct kvm_vcpu *vcpu,
513 struct sys_reg_params *p,
514 const struct sys_reg_desc *rd)
515{
516 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
517
518 if (p->is_write)
519 reg_to_dbg(vcpu, p, dbg_reg);
520 else
521 dbg_to_reg(vcpu, p, dbg_reg);
522
523 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
524
525 return true;
526}
527
528static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
529 const struct kvm_one_reg *reg, void __user *uaddr)
530{
531 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
532
533 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
534 return -EFAULT;
535 return 0;
536}
537
538static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
539 const struct kvm_one_reg *reg, void __user *uaddr)
540{
541 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
542
543 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
544 return -EFAULT;
545 return 0;
546}
547
548static void reset_wcr(struct kvm_vcpu *vcpu,
549 const struct sys_reg_desc *rd)
550{
551 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val;
552}
553
554static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
555{
556 u64 amair = read_sysreg(amair_el1);
557 vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
558}
559
560static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
561{
562 u64 mpidr;
563
564 /*
565 * Map the vcpu_id into the first three affinity level fields of
566 * the MPIDR. We limit the number of VCPUs in level 0 due to a
567 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
568 * of the GICv3 to be able to address each CPU directly when
569 * sending IPIs.
570 */
571 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
572 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
573 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
574 vcpu_write_sys_reg(vcpu, (1ULL << 31) | mpidr, MPIDR_EL1);
575}
576
577static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
578{
579 u64 pmcr, val;
580
581 pmcr = read_sysreg(pmcr_el0);
582 /*
583 * Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) are reset to UNKNOWN
584 * except PMCR.E resetting to zero.
585 */
586 val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
587 | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
588 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
589}
590
591static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
592{
593 u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
594 bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
595
596 if (!enabled)
597 kvm_inject_undefined(vcpu);
598
599 return !enabled;
600}
601
602static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
603{
604 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
605}
606
607static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
608{
609 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
610}
611
612static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
613{
614 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
615}
616
617static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
618{
619 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
620}
621
622static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
623 const struct sys_reg_desc *r)
624{
625 u64 val;
626
627 if (!kvm_arm_pmu_v3_ready(vcpu))
628 return trap_raz_wi(vcpu, p, r);
629
630 if (pmu_access_el0_disabled(vcpu))
631 return false;
632
633 if (p->is_write) {
634 /* Only update writeable bits of PMCR */
635 val = __vcpu_sys_reg(vcpu, PMCR_EL0);
636 val &= ~ARMV8_PMU_PMCR_MASK;
637 val |= p->regval & ARMV8_PMU_PMCR_MASK;
638 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
639 kvm_pmu_handle_pmcr(vcpu, val);
640 } else {
641 /* PMCR.P & PMCR.C are RAZ */
642 val = __vcpu_sys_reg(vcpu, PMCR_EL0)
643 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
644 p->regval = val;
645 }
646
647 return true;
648}
649
650static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
651 const struct sys_reg_desc *r)
652{
653 if (!kvm_arm_pmu_v3_ready(vcpu))
654 return trap_raz_wi(vcpu, p, r);
655
656 if (pmu_access_event_counter_el0_disabled(vcpu))
657 return false;
658
659 if (p->is_write)
660 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
661 else
662 /* return PMSELR.SEL field */
663 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
664 & ARMV8_PMU_COUNTER_MASK;
665
666 return true;
667}
668
669static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
670 const struct sys_reg_desc *r)
671{
672 u64 pmceid;
673
674 if (!kvm_arm_pmu_v3_ready(vcpu))
675 return trap_raz_wi(vcpu, p, r);
676
677 BUG_ON(p->is_write);
678
679 if (pmu_access_el0_disabled(vcpu))
680 return false;
681
682 if (!(p->Op2 & 1))
683 pmceid = read_sysreg(pmceid0_el0);
684 else
685 pmceid = read_sysreg(pmceid1_el0);
686
687 p->regval = pmceid;
688
689 return true;
690}
691
692static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
693{
694 u64 pmcr, val;
695
696 pmcr = __vcpu_sys_reg(vcpu, PMCR_EL0);
697 val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
698 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
699 kvm_inject_undefined(vcpu);
700 return false;
701 }
702
703 return true;
704}
705
706static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
707 struct sys_reg_params *p,
708 const struct sys_reg_desc *r)
709{
710 u64 idx;
711
712 if (!kvm_arm_pmu_v3_ready(vcpu))
713 return trap_raz_wi(vcpu, p, r);
714
715 if (r->CRn == 9 && r->CRm == 13) {
716 if (r->Op2 == 2) {
717 /* PMXEVCNTR_EL0 */
718 if (pmu_access_event_counter_el0_disabled(vcpu))
719 return false;
720
721 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
722 & ARMV8_PMU_COUNTER_MASK;
723 } else if (r->Op2 == 0) {
724 /* PMCCNTR_EL0 */
725 if (pmu_access_cycle_counter_el0_disabled(vcpu))
726 return false;
727
728 idx = ARMV8_PMU_CYCLE_IDX;
729 } else {
730 return false;
731 }
732 } else if (r->CRn == 0 && r->CRm == 9) {
733 /* PMCCNTR */
734 if (pmu_access_event_counter_el0_disabled(vcpu))
735 return false;
736
737 idx = ARMV8_PMU_CYCLE_IDX;
738 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
739 /* PMEVCNTRn_EL0 */
740 if (pmu_access_event_counter_el0_disabled(vcpu))
741 return false;
742
743 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
744 } else {
745 return false;
746 }
747
748 if (!pmu_counter_idx_valid(vcpu, idx))
749 return false;
750
751 if (p->is_write) {
752 if (pmu_access_el0_disabled(vcpu))
753 return false;
754
755 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
756 } else {
757 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
758 }
759
760 return true;
761}
762
763static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
764 const struct sys_reg_desc *r)
765{
766 u64 idx, reg;
767
768 if (!kvm_arm_pmu_v3_ready(vcpu))
769 return trap_raz_wi(vcpu, p, r);
770
771 if (pmu_access_el0_disabled(vcpu))
772 return false;
773
774 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
775 /* PMXEVTYPER_EL0 */
776 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
777 reg = PMEVTYPER0_EL0 + idx;
778 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
779 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
780 if (idx == ARMV8_PMU_CYCLE_IDX)
781 reg = PMCCFILTR_EL0;
782 else
783 /* PMEVTYPERn_EL0 */
784 reg = PMEVTYPER0_EL0 + idx;
785 } else {
786 BUG();
787 }
788
789 if (!pmu_counter_idx_valid(vcpu, idx))
790 return false;
791
792 if (p->is_write) {
793 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
794 __vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
795 } else {
796 p->regval = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
797 }
798
799 return true;
800}
801
802static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
803 const struct sys_reg_desc *r)
804{
805 u64 val, mask;
806
807 if (!kvm_arm_pmu_v3_ready(vcpu))
808 return trap_raz_wi(vcpu, p, r);
809
810 if (pmu_access_el0_disabled(vcpu))
811 return false;
812
813 mask = kvm_pmu_valid_counter_mask(vcpu);
814 if (p->is_write) {
815 val = p->regval & mask;
816 if (r->Op2 & 0x1) {
817 /* accessing PMCNTENSET_EL0 */
818 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
819 kvm_pmu_enable_counter(vcpu, val);
820 } else {
821 /* accessing PMCNTENCLR_EL0 */
822 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
823 kvm_pmu_disable_counter(vcpu, val);
824 }
825 } else {
826 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
827 }
828
829 return true;
830}
831
832static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
833 const struct sys_reg_desc *r)
834{
835 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
836
837 if (!kvm_arm_pmu_v3_ready(vcpu))
838 return trap_raz_wi(vcpu, p, r);
839
840 if (!vcpu_mode_priv(vcpu)) {
841 kvm_inject_undefined(vcpu);
842 return false;
843 }
844
845 if (p->is_write) {
846 u64 val = p->regval & mask;
847
848 if (r->Op2 & 0x1)
849 /* accessing PMINTENSET_EL1 */
850 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
851 else
852 /* accessing PMINTENCLR_EL1 */
853 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
854 } else {
855 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1) & mask;
856 }
857
858 return true;
859}
860
861static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
862 const struct sys_reg_desc *r)
863{
864 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
865
866 if (!kvm_arm_pmu_v3_ready(vcpu))
867 return trap_raz_wi(vcpu, p, r);
868
869 if (pmu_access_el0_disabled(vcpu))
870 return false;
871
872 if (p->is_write) {
873 if (r->CRm & 0x2)
874 /* accessing PMOVSSET_EL0 */
875 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
876 else
877 /* accessing PMOVSCLR_EL0 */
878 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
879 } else {
880 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0) & mask;
881 }
882
883 return true;
884}
885
886static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
887 const struct sys_reg_desc *r)
888{
889 u64 mask;
890
891 if (!kvm_arm_pmu_v3_ready(vcpu))
892 return trap_raz_wi(vcpu, p, r);
893
894 if (!p->is_write)
895 return read_from_write_only(vcpu, p, r);
896
897 if (pmu_write_swinc_el0_disabled(vcpu))
898 return false;
899
900 mask = kvm_pmu_valid_counter_mask(vcpu);
901 kvm_pmu_software_increment(vcpu, p->regval & mask);
902 return true;
903}
904
905static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
906 const struct sys_reg_desc *r)
907{
908 if (!kvm_arm_pmu_v3_ready(vcpu))
909 return trap_raz_wi(vcpu, p, r);
910
911 if (p->is_write) {
912 if (!vcpu_mode_priv(vcpu)) {
913 kvm_inject_undefined(vcpu);
914 return false;
915 }
916
917 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
918 p->regval & ARMV8_PMU_USERENR_MASK;
919 } else {
920 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
921 & ARMV8_PMU_USERENR_MASK;
922 }
923
924 return true;
925}
926
927/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
928#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
929 { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
930 trap_bvr, reset_bvr, n, 0, get_bvr, set_bvr }, \
931 { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
932 trap_bcr, reset_bcr, n, 0, get_bcr, set_bcr }, \
933 { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
934 trap_wvr, reset_wvr, n, 0, get_wvr, set_wvr }, \
935 { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
936 trap_wcr, reset_wcr, n, 0, get_wcr, set_wcr }
937
938/* Macro to expand the PMEVCNTRn_EL0 register */
939#define PMU_PMEVCNTR_EL0(n) \
940 { SYS_DESC(SYS_PMEVCNTRn_EL0(n)), \
941 access_pmu_evcntr, reset_unknown, (PMEVCNTR0_EL0 + n), }
942
943/* Macro to expand the PMEVTYPERn_EL0 register */
944#define PMU_PMEVTYPER_EL0(n) \
945 { SYS_DESC(SYS_PMEVTYPERn_EL0(n)), \
946 access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
947
948static bool access_cntp_tval(struct kvm_vcpu *vcpu,
949 struct sys_reg_params *p,
950 const struct sys_reg_desc *r)
951{
952 u64 now = kvm_phys_timer_read();
953 u64 cval;
954
955 if (p->is_write) {
956 kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL,
957 p->regval + now);
958 } else {
959 cval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL);
960 p->regval = cval - now;
961 }
962
963 return true;
964}
965
966static bool access_cntp_ctl(struct kvm_vcpu *vcpu,
967 struct sys_reg_params *p,
968 const struct sys_reg_desc *r)
969{
970 if (p->is_write)
971 kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CTL, p->regval);
972 else
973 p->regval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CTL);
974
975 return true;
976}
977
978static bool access_cntp_cval(struct kvm_vcpu *vcpu,
979 struct sys_reg_params *p,
980 const struct sys_reg_desc *r)
981{
982 if (p->is_write)
983 kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL, p->regval);
984 else
985 p->regval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL);
986
987 return true;
988}
989
990/* Read a sanitised cpufeature ID register by sys_reg_desc */
991static u64 read_id_reg(struct sys_reg_desc const *r, bool raz)
992{
993 u32 id = sys_reg((u32)r->Op0, (u32)r->Op1,
994 (u32)r->CRn, (u32)r->CRm, (u32)r->Op2);
995 u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
996
997 if (id == SYS_ID_AA64PFR0_EL1) {
998 if (val & (0xfUL << ID_AA64PFR0_SVE_SHIFT))
999 kvm_debug("SVE unsupported for guests, suppressing\n");
1000
1001 val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
1002 } else if (id == SYS_ID_AA64MMFR1_EL1) {
1003 if (val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))
1004 kvm_debug("LORegions unsupported for guests, suppressing\n");
1005
1006 val &= ~(0xfUL << ID_AA64MMFR1_LOR_SHIFT);
1007 }
1008
1009 return val;
1010}
1011
1012/* cpufeature ID register access trap handlers */
1013
1014static bool __access_id_reg(struct kvm_vcpu *vcpu,
1015 struct sys_reg_params *p,
1016 const struct sys_reg_desc *r,
1017 bool raz)
1018{
1019 if (p->is_write)
1020 return write_to_read_only(vcpu, p, r);
1021
1022 p->regval = read_id_reg(r, raz);
1023 return true;
1024}
1025
1026static bool access_id_reg(struct kvm_vcpu *vcpu,
1027 struct sys_reg_params *p,
1028 const struct sys_reg_desc *r)
1029{
1030 return __access_id_reg(vcpu, p, r, false);
1031}
1032
1033static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
1034 struct sys_reg_params *p,
1035 const struct sys_reg_desc *r)
1036{
1037 return __access_id_reg(vcpu, p, r, true);
1038}
1039
1040static int reg_from_user(u64 *val, const void __user *uaddr, u64 id);
1041static int reg_to_user(void __user *uaddr, const u64 *val, u64 id);
1042static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
1043
1044/*
1045 * cpufeature ID register user accessors
1046 *
1047 * For now, these registers are immutable for userspace, so no values
1048 * are stored, and for set_id_reg() we don't allow the effective value
1049 * to be changed.
1050 */
1051static int __get_id_reg(const struct sys_reg_desc *rd, void __user *uaddr,
1052 bool raz)
1053{
1054 const u64 id = sys_reg_to_index(rd);
1055 const u64 val = read_id_reg(rd, raz);
1056
1057 return reg_to_user(uaddr, &val, id);
1058}
1059
1060static int __set_id_reg(const struct sys_reg_desc *rd, void __user *uaddr,
1061 bool raz)
1062{
1063 const u64 id = sys_reg_to_index(rd);
1064 int err;
1065 u64 val;
1066
1067 err = reg_from_user(&val, uaddr, id);
1068 if (err)
1069 return err;
1070
1071 /* This is what we mean by invariant: you can't change it. */
1072 if (val != read_id_reg(rd, raz))
1073 return -EINVAL;
1074
1075 return 0;
1076}
1077
1078static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1079 const struct kvm_one_reg *reg, void __user *uaddr)
1080{
1081 return __get_id_reg(rd, uaddr, false);
1082}
1083
1084static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1085 const struct kvm_one_reg *reg, void __user *uaddr)
1086{
1087 return __set_id_reg(rd, uaddr, false);
1088}
1089
1090static int get_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1091 const struct kvm_one_reg *reg, void __user *uaddr)
1092{
1093 return __get_id_reg(rd, uaddr, true);
1094}
1095
1096static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1097 const struct kvm_one_reg *reg, void __user *uaddr)
1098{
1099 return __set_id_reg(rd, uaddr, true);
1100}
1101
1102/* sys_reg_desc initialiser for known cpufeature ID registers */
1103#define ID_SANITISED(name) { \
1104 SYS_DESC(SYS_##name), \
1105 .access = access_id_reg, \
1106 .get_user = get_id_reg, \
1107 .set_user = set_id_reg, \
1108}
1109
1110/*
1111 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
1112 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
1113 * (1 <= crm < 8, 0 <= Op2 < 8).
1114 */
1115#define ID_UNALLOCATED(crm, op2) { \
1116 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
1117 .access = access_raz_id_reg, \
1118 .get_user = get_raz_id_reg, \
1119 .set_user = set_raz_id_reg, \
1120}
1121
1122/*
1123 * sys_reg_desc initialiser for known ID registers that we hide from guests.
1124 * For now, these are exposed just like unallocated ID regs: they appear
1125 * RAZ for the guest.
1126 */
1127#define ID_HIDDEN(name) { \
1128 SYS_DESC(SYS_##name), \
1129 .access = access_raz_id_reg, \
1130 .get_user = get_raz_id_reg, \
1131 .set_user = set_raz_id_reg, \
1132}
1133
1134/*
1135 * Architected system registers.
1136 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
1137 *
1138 * Debug handling: We do trap most, if not all debug related system
1139 * registers. The implementation is good enough to ensure that a guest
1140 * can use these with minimal performance degradation. The drawback is
1141 * that we don't implement any of the external debug, none of the
1142 * OSlock protocol. This should be revisited if we ever encounter a
1143 * more demanding guest...
1144 */
1145static const struct sys_reg_desc sys_reg_descs[] = {
1146 { SYS_DESC(SYS_DC_ISW), access_dcsw },
1147 { SYS_DESC(SYS_DC_CSW), access_dcsw },
1148 { SYS_DESC(SYS_DC_CISW), access_dcsw },
1149
1150 DBG_BCR_BVR_WCR_WVR_EL1(0),
1151 DBG_BCR_BVR_WCR_WVR_EL1(1),
1152 { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
1153 { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
1154 DBG_BCR_BVR_WCR_WVR_EL1(2),
1155 DBG_BCR_BVR_WCR_WVR_EL1(3),
1156 DBG_BCR_BVR_WCR_WVR_EL1(4),
1157 DBG_BCR_BVR_WCR_WVR_EL1(5),
1158 DBG_BCR_BVR_WCR_WVR_EL1(6),
1159 DBG_BCR_BVR_WCR_WVR_EL1(7),
1160 DBG_BCR_BVR_WCR_WVR_EL1(8),
1161 DBG_BCR_BVR_WCR_WVR_EL1(9),
1162 DBG_BCR_BVR_WCR_WVR_EL1(10),
1163 DBG_BCR_BVR_WCR_WVR_EL1(11),
1164 DBG_BCR_BVR_WCR_WVR_EL1(12),
1165 DBG_BCR_BVR_WCR_WVR_EL1(13),
1166 DBG_BCR_BVR_WCR_WVR_EL1(14),
1167 DBG_BCR_BVR_WCR_WVR_EL1(15),
1168
1169 { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
1170 { SYS_DESC(SYS_OSLAR_EL1), trap_raz_wi },
1171 { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1 },
1172 { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
1173 { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
1174 { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
1175 { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
1176 { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
1177
1178 { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
1179 { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
1180 // DBGDTR[TR]X_EL0 share the same encoding
1181 { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
1182
1183 { SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
1184
1185 { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
1186
1187 /*
1188 * ID regs: all ID_SANITISED() entries here must have corresponding
1189 * entries in arm64_ftr_regs[].
1190 */
1191
1192 /* AArch64 mappings of the AArch32 ID registers */
1193 /* CRm=1 */
1194 ID_SANITISED(ID_PFR0_EL1),
1195 ID_SANITISED(ID_PFR1_EL1),
1196 ID_SANITISED(ID_DFR0_EL1),
1197 ID_HIDDEN(ID_AFR0_EL1),
1198 ID_SANITISED(ID_MMFR0_EL1),
1199 ID_SANITISED(ID_MMFR1_EL1),
1200 ID_SANITISED(ID_MMFR2_EL1),
1201 ID_SANITISED(ID_MMFR3_EL1),
1202
1203 /* CRm=2 */
1204 ID_SANITISED(ID_ISAR0_EL1),
1205 ID_SANITISED(ID_ISAR1_EL1),
1206 ID_SANITISED(ID_ISAR2_EL1),
1207 ID_SANITISED(ID_ISAR3_EL1),
1208 ID_SANITISED(ID_ISAR4_EL1),
1209 ID_SANITISED(ID_ISAR5_EL1),
1210 ID_SANITISED(ID_MMFR4_EL1),
1211 ID_UNALLOCATED(2,7),
1212
1213 /* CRm=3 */
1214 ID_SANITISED(MVFR0_EL1),
1215 ID_SANITISED(MVFR1_EL1),
1216 ID_SANITISED(MVFR2_EL1),
1217 ID_UNALLOCATED(3,3),
1218 ID_UNALLOCATED(3,4),
1219 ID_UNALLOCATED(3,5),
1220 ID_UNALLOCATED(3,6),
1221 ID_UNALLOCATED(3,7),
1222
1223 /* AArch64 ID registers */
1224 /* CRm=4 */
1225 ID_SANITISED(ID_AA64PFR0_EL1),
1226 ID_SANITISED(ID_AA64PFR1_EL1),
1227 ID_UNALLOCATED(4,2),
1228 ID_UNALLOCATED(4,3),
1229 ID_UNALLOCATED(4,4),
1230 ID_UNALLOCATED(4,5),
1231 ID_UNALLOCATED(4,6),
1232 ID_UNALLOCATED(4,7),
1233
1234 /* CRm=5 */
1235 ID_SANITISED(ID_AA64DFR0_EL1),
1236 ID_SANITISED(ID_AA64DFR1_EL1),
1237 ID_UNALLOCATED(5,2),
1238 ID_UNALLOCATED(5,3),
1239 ID_HIDDEN(ID_AA64AFR0_EL1),
1240 ID_HIDDEN(ID_AA64AFR1_EL1),
1241 ID_UNALLOCATED(5,6),
1242 ID_UNALLOCATED(5,7),
1243
1244 /* CRm=6 */
1245 ID_SANITISED(ID_AA64ISAR0_EL1),
1246 ID_SANITISED(ID_AA64ISAR1_EL1),
1247 ID_UNALLOCATED(6,2),
1248 ID_UNALLOCATED(6,3),
1249 ID_UNALLOCATED(6,4),
1250 ID_UNALLOCATED(6,5),
1251 ID_UNALLOCATED(6,6),
1252 ID_UNALLOCATED(6,7),
1253
1254 /* CRm=7 */
1255 ID_SANITISED(ID_AA64MMFR0_EL1),
1256 ID_SANITISED(ID_AA64MMFR1_EL1),
1257 ID_SANITISED(ID_AA64MMFR2_EL1),
1258 ID_UNALLOCATED(7,3),
1259 ID_UNALLOCATED(7,4),
1260 ID_UNALLOCATED(7,5),
1261 ID_UNALLOCATED(7,6),
1262 ID_UNALLOCATED(7,7),
1263
1264 { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
1265 { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
1266 { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
1267 { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
1268 { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
1269
1270 { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
1271 { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
1272 { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
1273
1274 { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
1275 { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
1276 { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
1277 { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
1278 { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
1279 { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
1280 { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
1281 { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
1282
1283 { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
1284 { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
1285
1286 { SYS_DESC(SYS_PMINTENSET_EL1), access_pminten, reset_unknown, PMINTENSET_EL1 },
1287 { SYS_DESC(SYS_PMINTENCLR_EL1), access_pminten, NULL, PMINTENSET_EL1 },
1288
1289 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
1290 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
1291
1292 { SYS_DESC(SYS_LORSA_EL1), trap_undef },
1293 { SYS_DESC(SYS_LOREA_EL1), trap_undef },
1294 { SYS_DESC(SYS_LORN_EL1), trap_undef },
1295 { SYS_DESC(SYS_LORC_EL1), trap_undef },
1296 { SYS_DESC(SYS_LORID_EL1), trap_undef },
1297
1298 { SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
1299 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
1300
1301 { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
1302 { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
1303 { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
1304 { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
1305 { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
1306 { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
1307 { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
1308 { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
1309 { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
1310 { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
1311
1312 { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
1313 { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
1314
1315 { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
1316
1317 { SYS_DESC(SYS_CSSELR_EL1), NULL, reset_unknown, CSSELR_EL1 },
1318
1319 { SYS_DESC(SYS_PMCR_EL0), access_pmcr, reset_pmcr, },
1320 { SYS_DESC(SYS_PMCNTENSET_EL0), access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
1321 { SYS_DESC(SYS_PMCNTENCLR_EL0), access_pmcnten, NULL, PMCNTENSET_EL0 },
1322 { SYS_DESC(SYS_PMOVSCLR_EL0), access_pmovs, NULL, PMOVSSET_EL0 },
1323 { SYS_DESC(SYS_PMSWINC_EL0), access_pmswinc, reset_unknown, PMSWINC_EL0 },
1324 { SYS_DESC(SYS_PMSELR_EL0), access_pmselr, reset_unknown, PMSELR_EL0 },
1325 { SYS_DESC(SYS_PMCEID0_EL0), access_pmceid },
1326 { SYS_DESC(SYS_PMCEID1_EL0), access_pmceid },
1327 { SYS_DESC(SYS_PMCCNTR_EL0), access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
1328 { SYS_DESC(SYS_PMXEVTYPER_EL0), access_pmu_evtyper },
1329 { SYS_DESC(SYS_PMXEVCNTR_EL0), access_pmu_evcntr },
1330 /*
1331 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
1332 * in 32bit mode. Here we choose to reset it as zero for consistency.
1333 */
1334 { SYS_DESC(SYS_PMUSERENR_EL0), access_pmuserenr, reset_val, PMUSERENR_EL0, 0 },
1335 { SYS_DESC(SYS_PMOVSSET_EL0), access_pmovs, reset_unknown, PMOVSSET_EL0 },
1336
1337 { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
1338 { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
1339
1340 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_cntp_tval },
1341 { SYS_DESC(SYS_CNTP_CTL_EL0), access_cntp_ctl },
1342 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_cntp_cval },
1343
1344 /* PMEVCNTRn_EL0 */
1345 PMU_PMEVCNTR_EL0(0),
1346 PMU_PMEVCNTR_EL0(1),
1347 PMU_PMEVCNTR_EL0(2),
1348 PMU_PMEVCNTR_EL0(3),
1349 PMU_PMEVCNTR_EL0(4),
1350 PMU_PMEVCNTR_EL0(5),
1351 PMU_PMEVCNTR_EL0(6),
1352 PMU_PMEVCNTR_EL0(7),
1353 PMU_PMEVCNTR_EL0(8),
1354 PMU_PMEVCNTR_EL0(9),
1355 PMU_PMEVCNTR_EL0(10),
1356 PMU_PMEVCNTR_EL0(11),
1357 PMU_PMEVCNTR_EL0(12),
1358 PMU_PMEVCNTR_EL0(13),
1359 PMU_PMEVCNTR_EL0(14),
1360 PMU_PMEVCNTR_EL0(15),
1361 PMU_PMEVCNTR_EL0(16),
1362 PMU_PMEVCNTR_EL0(17),
1363 PMU_PMEVCNTR_EL0(18),
1364 PMU_PMEVCNTR_EL0(19),
1365 PMU_PMEVCNTR_EL0(20),
1366 PMU_PMEVCNTR_EL0(21),
1367 PMU_PMEVCNTR_EL0(22),
1368 PMU_PMEVCNTR_EL0(23),
1369 PMU_PMEVCNTR_EL0(24),
1370 PMU_PMEVCNTR_EL0(25),
1371 PMU_PMEVCNTR_EL0(26),
1372 PMU_PMEVCNTR_EL0(27),
1373 PMU_PMEVCNTR_EL0(28),
1374 PMU_PMEVCNTR_EL0(29),
1375 PMU_PMEVCNTR_EL0(30),
1376 /* PMEVTYPERn_EL0 */
1377 PMU_PMEVTYPER_EL0(0),
1378 PMU_PMEVTYPER_EL0(1),
1379 PMU_PMEVTYPER_EL0(2),
1380 PMU_PMEVTYPER_EL0(3),
1381 PMU_PMEVTYPER_EL0(4),
1382 PMU_PMEVTYPER_EL0(5),
1383 PMU_PMEVTYPER_EL0(6),
1384 PMU_PMEVTYPER_EL0(7),
1385 PMU_PMEVTYPER_EL0(8),
1386 PMU_PMEVTYPER_EL0(9),
1387 PMU_PMEVTYPER_EL0(10),
1388 PMU_PMEVTYPER_EL0(11),
1389 PMU_PMEVTYPER_EL0(12),
1390 PMU_PMEVTYPER_EL0(13),
1391 PMU_PMEVTYPER_EL0(14),
1392 PMU_PMEVTYPER_EL0(15),
1393 PMU_PMEVTYPER_EL0(16),
1394 PMU_PMEVTYPER_EL0(17),
1395 PMU_PMEVTYPER_EL0(18),
1396 PMU_PMEVTYPER_EL0(19),
1397 PMU_PMEVTYPER_EL0(20),
1398 PMU_PMEVTYPER_EL0(21),
1399 PMU_PMEVTYPER_EL0(22),
1400 PMU_PMEVTYPER_EL0(23),
1401 PMU_PMEVTYPER_EL0(24),
1402 PMU_PMEVTYPER_EL0(25),
1403 PMU_PMEVTYPER_EL0(26),
1404 PMU_PMEVTYPER_EL0(27),
1405 PMU_PMEVTYPER_EL0(28),
1406 PMU_PMEVTYPER_EL0(29),
1407 PMU_PMEVTYPER_EL0(30),
1408 /*
1409 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
1410 * in 32bit mode. Here we choose to reset it as zero for consistency.
1411 */
1412 { SYS_DESC(SYS_PMCCFILTR_EL0), access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },
1413
1414 { SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
1415 { SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
1416 { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x70 },
1417};
1418
1419static bool trap_dbgidr(struct kvm_vcpu *vcpu,
1420 struct sys_reg_params *p,
1421 const struct sys_reg_desc *r)
1422{
1423 if (p->is_write) {
1424 return ignore_write(vcpu, p);
1425 } else {
1426 u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1427 u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1428 u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
1429
1430 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
1431 (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
1432 (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1433 | (6 << 16) | (el3 << 14) | (el3 << 12));
1434 return true;
1435 }
1436}
1437
1438static bool trap_debug32(struct kvm_vcpu *vcpu,
1439 struct sys_reg_params *p,
1440 const struct sys_reg_desc *r)
1441{
1442 if (p->is_write) {
1443 vcpu_cp14(vcpu, r->reg) = p->regval;
1444 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
1445 } else {
1446 p->regval = vcpu_cp14(vcpu, r->reg);
1447 }
1448
1449 return true;
1450}
1451
1452/* AArch32 debug register mappings
1453 *
1454 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
1455 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
1456 *
1457 * All control registers and watchpoint value registers are mapped to
1458 * the lower 32 bits of their AArch64 equivalents. We share the trap
1459 * handlers with the above AArch64 code which checks what mode the
1460 * system is in.
1461 */
1462
1463static bool trap_xvr(struct kvm_vcpu *vcpu,
1464 struct sys_reg_params *p,
1465 const struct sys_reg_desc *rd)
1466{
1467 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
1468
1469 if (p->is_write) {
1470 u64 val = *dbg_reg;
1471
1472 val &= 0xffffffffUL;
1473 val |= p->regval << 32;
1474 *dbg_reg = val;
1475
1476 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
1477 } else {
1478 p->regval = *dbg_reg >> 32;
1479 }
1480
1481 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
1482
1483 return true;
1484}
1485
1486#define DBG_BCR_BVR_WCR_WVR(n) \
1487 /* DBGBVRn */ \
1488 { Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
1489 /* DBGBCRn */ \
1490 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
1491 /* DBGWVRn */ \
1492 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
1493 /* DBGWCRn */ \
1494 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
1495
1496#define DBGBXVR(n) \
1497 { Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_xvr, NULL, n }
1498
1499/*
1500 * Trapped cp14 registers. We generally ignore most of the external
1501 * debug, on the principle that they don't really make sense to a
1502 * guest. Revisit this one day, would this principle change.
1503 */
1504static const struct sys_reg_desc cp14_regs[] = {
1505 /* DBGIDR */
1506 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
1507 /* DBGDTRRXext */
1508 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1509
1510 DBG_BCR_BVR_WCR_WVR(0),
1511 /* DBGDSCRint */
1512 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
1513 DBG_BCR_BVR_WCR_WVR(1),
1514 /* DBGDCCINT */
1515 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug32 },
1516 /* DBGDSCRext */
1517 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug32 },
1518 DBG_BCR_BVR_WCR_WVR(2),
1519 /* DBGDTR[RT]Xint */
1520 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
1521 /* DBGDTR[RT]Xext */
1522 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
1523 DBG_BCR_BVR_WCR_WVR(3),
1524 DBG_BCR_BVR_WCR_WVR(4),
1525 DBG_BCR_BVR_WCR_WVR(5),
1526 /* DBGWFAR */
1527 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
1528 /* DBGOSECCR */
1529 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
1530 DBG_BCR_BVR_WCR_WVR(6),
1531 /* DBGVCR */
1532 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug32 },
1533 DBG_BCR_BVR_WCR_WVR(7),
1534 DBG_BCR_BVR_WCR_WVR(8),
1535 DBG_BCR_BVR_WCR_WVR(9),
1536 DBG_BCR_BVR_WCR_WVR(10),
1537 DBG_BCR_BVR_WCR_WVR(11),
1538 DBG_BCR_BVR_WCR_WVR(12),
1539 DBG_BCR_BVR_WCR_WVR(13),
1540 DBG_BCR_BVR_WCR_WVR(14),
1541 DBG_BCR_BVR_WCR_WVR(15),
1542
1543 /* DBGDRAR (32bit) */
1544 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
1545
1546 DBGBXVR(0),
1547 /* DBGOSLAR */
1548 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
1549 DBGBXVR(1),
1550 /* DBGOSLSR */
1551 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
1552 DBGBXVR(2),
1553 DBGBXVR(3),
1554 /* DBGOSDLR */
1555 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
1556 DBGBXVR(4),
1557 /* DBGPRCR */
1558 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
1559 DBGBXVR(5),
1560 DBGBXVR(6),
1561 DBGBXVR(7),
1562 DBGBXVR(8),
1563 DBGBXVR(9),
1564 DBGBXVR(10),
1565 DBGBXVR(11),
1566 DBGBXVR(12),
1567 DBGBXVR(13),
1568 DBGBXVR(14),
1569 DBGBXVR(15),
1570
1571 /* DBGDSAR (32bit) */
1572 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
1573
1574 /* DBGDEVID2 */
1575 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
1576 /* DBGDEVID1 */
1577 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
1578 /* DBGDEVID */
1579 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
1580 /* DBGCLAIMSET */
1581 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
1582 /* DBGCLAIMCLR */
1583 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
1584 /* DBGAUTHSTATUS */
1585 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
1586};
1587
1588/* Trapped cp14 64bit registers */
1589static const struct sys_reg_desc cp14_64_regs[] = {
1590 /* DBGDRAR (64bit) */
1591 { Op1( 0), CRm( 1), .access = trap_raz_wi },
1592
1593 /* DBGDSAR (64bit) */
1594 { Op1( 0), CRm( 2), .access = trap_raz_wi },
1595};
1596
1597/* Macro to expand the PMEVCNTRn register */
1598#define PMU_PMEVCNTR(n) \
1599 /* PMEVCNTRn */ \
1600 { Op1(0), CRn(0b1110), \
1601 CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1602 access_pmu_evcntr }
1603
1604/* Macro to expand the PMEVTYPERn register */
1605#define PMU_PMEVTYPER(n) \
1606 /* PMEVTYPERn */ \
1607 { Op1(0), CRn(0b1110), \
1608 CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1609 access_pmu_evtyper }
1610
1611/*
1612 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
1613 * depending on the way they are accessed (as a 32bit or a 64bit
1614 * register).
1615 */
1616static const struct sys_reg_desc cp15_regs[] = {
1617 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
1618
1619 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR },
1620 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
1621 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
1622 { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR },
1623 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, c3_DACR },
1624 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, c5_DFSR },
1625 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, c5_IFSR },
1626 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, c5_ADFSR },
1627 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, c5_AIFSR },
1628 { Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, c6_DFAR },
1629 { Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, c6_IFAR },
1630
1631 /*
1632 * DC{C,I,CI}SW operations:
1633 */
1634 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
1635 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
1636 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
1637
1638 /* PMU */
1639 { Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr },
1640 { Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten },
1641 { Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten },
1642 { Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
1643 { Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
1644 { Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
1645 { Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1646 { Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
1647 { Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
1648 { Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
1649 { Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
1650 { Op1( 0), CRn( 9), CRm(14), Op2( 0), access_pmuserenr },
1651 { Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1652 { Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
1653 { Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
1654
1655 { Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, c10_PRRR },
1656 { Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR },
1657 { Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 },
1658 { Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 },
1659
1660 /* ICC_SRE */
1661 { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
1662
1663 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID },
1664
1665 /* CNTP_TVAL */
1666 { Op1( 0), CRn(14), CRm( 2), Op2( 0), access_cntp_tval },
1667 /* CNTP_CTL */
1668 { Op1( 0), CRn(14), CRm( 2), Op2( 1), access_cntp_ctl },
1669
1670 /* PMEVCNTRn */
1671 PMU_PMEVCNTR(0),
1672 PMU_PMEVCNTR(1),
1673 PMU_PMEVCNTR(2),
1674 PMU_PMEVCNTR(3),
1675 PMU_PMEVCNTR(4),
1676 PMU_PMEVCNTR(5),
1677 PMU_PMEVCNTR(6),
1678 PMU_PMEVCNTR(7),
1679 PMU_PMEVCNTR(8),
1680 PMU_PMEVCNTR(9),
1681 PMU_PMEVCNTR(10),
1682 PMU_PMEVCNTR(11),
1683 PMU_PMEVCNTR(12),
1684 PMU_PMEVCNTR(13),
1685 PMU_PMEVCNTR(14),
1686 PMU_PMEVCNTR(15),
1687 PMU_PMEVCNTR(16),
1688 PMU_PMEVCNTR(17),
1689 PMU_PMEVCNTR(18),
1690 PMU_PMEVCNTR(19),
1691 PMU_PMEVCNTR(20),
1692 PMU_PMEVCNTR(21),
1693 PMU_PMEVCNTR(22),
1694 PMU_PMEVCNTR(23),
1695 PMU_PMEVCNTR(24),
1696 PMU_PMEVCNTR(25),
1697 PMU_PMEVCNTR(26),
1698 PMU_PMEVCNTR(27),
1699 PMU_PMEVCNTR(28),
1700 PMU_PMEVCNTR(29),
1701 PMU_PMEVCNTR(30),
1702 /* PMEVTYPERn */
1703 PMU_PMEVTYPER(0),
1704 PMU_PMEVTYPER(1),
1705 PMU_PMEVTYPER(2),
1706 PMU_PMEVTYPER(3),
1707 PMU_PMEVTYPER(4),
1708 PMU_PMEVTYPER(5),
1709 PMU_PMEVTYPER(6),
1710 PMU_PMEVTYPER(7),
1711 PMU_PMEVTYPER(8),
1712 PMU_PMEVTYPER(9),
1713 PMU_PMEVTYPER(10),
1714 PMU_PMEVTYPER(11),
1715 PMU_PMEVTYPER(12),
1716 PMU_PMEVTYPER(13),
1717 PMU_PMEVTYPER(14),
1718 PMU_PMEVTYPER(15),
1719 PMU_PMEVTYPER(16),
1720 PMU_PMEVTYPER(17),
1721 PMU_PMEVTYPER(18),
1722 PMU_PMEVTYPER(19),
1723 PMU_PMEVTYPER(20),
1724 PMU_PMEVTYPER(21),
1725 PMU_PMEVTYPER(22),
1726 PMU_PMEVTYPER(23),
1727 PMU_PMEVTYPER(24),
1728 PMU_PMEVTYPER(25),
1729 PMU_PMEVTYPER(26),
1730 PMU_PMEVTYPER(27),
1731 PMU_PMEVTYPER(28),
1732 PMU_PMEVTYPER(29),
1733 PMU_PMEVTYPER(30),
1734 /* PMCCFILTR */
1735 { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
1736};
1737
1738static const struct sys_reg_desc cp15_64_regs[] = {
1739 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
1740 { Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
1741 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
1742 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
1743 { Op1( 2), CRn( 0), CRm(14), Op2( 0), access_cntp_cval },
1744};
1745
1746/* Target specific emulation tables */
1747static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS];
1748
1749void kvm_register_target_sys_reg_table(unsigned int target,
1750 struct kvm_sys_reg_target_table *table)
1751{
1752 target_tables[target] = table;
1753}
1754
1755/* Get specific register table for this target. */
1756static const struct sys_reg_desc *get_target_table(unsigned target,
1757 bool mode_is_64,
1758 size_t *num)
1759{
1760 struct kvm_sys_reg_target_table *table;
1761
1762 table = target_tables[target];
1763 if (mode_is_64) {
1764 *num = table->table64.num;
1765 return table->table64.table;
1766 } else {
1767 *num = table->table32.num;
1768 return table->table32.table;
1769 }
1770}
1771
1772#define reg_to_match_value(x) \
1773 ({ \
1774 unsigned long val; \
1775 val = (x)->Op0 << 14; \
1776 val |= (x)->Op1 << 11; \
1777 val |= (x)->CRn << 7; \
1778 val |= (x)->CRm << 3; \
1779 val |= (x)->Op2; \
1780 val; \
1781 })
1782
1783static int match_sys_reg(const void *key, const void *elt)
1784{
1785 const unsigned long pval = (unsigned long)key;
1786 const struct sys_reg_desc *r = elt;
1787
1788 return pval - reg_to_match_value(r);
1789}
1790
1791static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
1792 const struct sys_reg_desc table[],
1793 unsigned int num)
1794{
1795 unsigned long pval = reg_to_match_value(params);
1796
1797 return bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg);
1798}
1799
1800int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
1801{
1802 kvm_inject_undefined(vcpu);
1803 return 1;
1804}
1805
1806static void perform_access(struct kvm_vcpu *vcpu,
1807 struct sys_reg_params *params,
1808 const struct sys_reg_desc *r)
1809{
1810 /*
1811 * Not having an accessor means that we have configured a trap
1812 * that we don't know how to handle. This certainly qualifies
1813 * as a gross bug that should be fixed right away.
1814 */
1815 BUG_ON(!r->access);
1816
1817 /* Skip instruction if instructed so */
1818 if (likely(r->access(vcpu, params, r)))
1819 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
1820}
1821
1822/*
1823 * emulate_cp -- tries to match a sys_reg access in a handling table, and
1824 * call the corresponding trap handler.
1825 *
1826 * @params: pointer to the descriptor of the access
1827 * @table: array of trap descriptors
1828 * @num: size of the trap descriptor array
1829 *
1830 * Return 0 if the access has been handled, and -1 if not.
1831 */
1832static int emulate_cp(struct kvm_vcpu *vcpu,
1833 struct sys_reg_params *params,
1834 const struct sys_reg_desc *table,
1835 size_t num)
1836{
1837 const struct sys_reg_desc *r;
1838
1839 if (!table)
1840 return -1; /* Not handled */
1841
1842 r = find_reg(params, table, num);
1843
1844 if (r) {
1845 perform_access(vcpu, params, r);
1846 return 0;
1847 }
1848
1849 /* Not handled */
1850 return -1;
1851}
1852
1853static void unhandled_cp_access(struct kvm_vcpu *vcpu,
1854 struct sys_reg_params *params)
1855{
1856 u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
1857 int cp = -1;
1858
1859 switch(hsr_ec) {
1860 case ESR_ELx_EC_CP15_32:
1861 case ESR_ELx_EC_CP15_64:
1862 cp = 15;
1863 break;
1864 case ESR_ELx_EC_CP14_MR:
1865 case ESR_ELx_EC_CP14_64:
1866 cp = 14;
1867 break;
1868 default:
1869 WARN_ON(1);
1870 }
1871
1872 kvm_err("Unsupported guest CP%d access at: %08lx\n",
1873 cp, *vcpu_pc(vcpu));
1874 print_sys_reg_instr(params);
1875 kvm_inject_undefined(vcpu);
1876}
1877
1878/**
1879 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
1880 * @vcpu: The VCPU pointer
1881 * @run: The kvm_run struct
1882 */
1883static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
1884 const struct sys_reg_desc *global,
1885 size_t nr_global,
1886 const struct sys_reg_desc *target_specific,
1887 size_t nr_specific)
1888{
1889 struct sys_reg_params params;
1890 u32 hsr = kvm_vcpu_get_hsr(vcpu);
1891 int Rt = kvm_vcpu_sys_get_rt(vcpu);
1892 int Rt2 = (hsr >> 10) & 0x1f;
1893
1894 params.is_aarch32 = true;
1895 params.is_32bit = false;
1896 params.CRm = (hsr >> 1) & 0xf;
1897 params.is_write = ((hsr & 1) == 0);
1898
1899 params.Op0 = 0;
1900 params.Op1 = (hsr >> 16) & 0xf;
1901 params.Op2 = 0;
1902 params.CRn = 0;
1903
1904 /*
1905 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
1906 * backends between AArch32 and AArch64, we get away with it.
1907 */
1908 if (params.is_write) {
1909 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
1910 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
1911 }
1912
1913 /*
1914 * Try to emulate the coprocessor access using the target
1915 * specific table first, and using the global table afterwards.
1916 * If either of the tables contains a handler, handle the
1917 * potential register operation in the case of a read and return
1918 * with success.
1919 */
1920 if (!emulate_cp(vcpu, ¶ms, target_specific, nr_specific) ||
1921 !emulate_cp(vcpu, ¶ms, global, nr_global)) {
1922 /* Split up the value between registers for the read side */
1923 if (!params.is_write) {
1924 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
1925 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
1926 }
1927
1928 return 1;
1929 }
1930
1931 unhandled_cp_access(vcpu, ¶ms);
1932 return 1;
1933}
1934
1935/**
1936 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
1937 * @vcpu: The VCPU pointer
1938 * @run: The kvm_run struct
1939 */
1940static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
1941 const struct sys_reg_desc *global,
1942 size_t nr_global,
1943 const struct sys_reg_desc *target_specific,
1944 size_t nr_specific)
1945{
1946 struct sys_reg_params params;
1947 u32 hsr = kvm_vcpu_get_hsr(vcpu);
1948 int Rt = kvm_vcpu_sys_get_rt(vcpu);
1949
1950 params.is_aarch32 = true;
1951 params.is_32bit = true;
1952 params.CRm = (hsr >> 1) & 0xf;
1953 params.regval = vcpu_get_reg(vcpu, Rt);
1954 params.is_write = ((hsr & 1) == 0);
1955 params.CRn = (hsr >> 10) & 0xf;
1956 params.Op0 = 0;
1957 params.Op1 = (hsr >> 14) & 0x7;
1958 params.Op2 = (hsr >> 17) & 0x7;
1959
1960 if (!emulate_cp(vcpu, ¶ms, target_specific, nr_specific) ||
1961 !emulate_cp(vcpu, ¶ms, global, nr_global)) {
1962 if (!params.is_write)
1963 vcpu_set_reg(vcpu, Rt, params.regval);
1964 return 1;
1965 }
1966
1967 unhandled_cp_access(vcpu, ¶ms);
1968 return 1;
1969}
1970
1971int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
1972{
1973 const struct sys_reg_desc *target_specific;
1974 size_t num;
1975
1976 target_specific = get_target_table(vcpu->arch.target, false, &num);
1977 return kvm_handle_cp_64(vcpu,
1978 cp15_64_regs, ARRAY_SIZE(cp15_64_regs),
1979 target_specific, num);
1980}
1981
1982int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
1983{
1984 const struct sys_reg_desc *target_specific;
1985 size_t num;
1986
1987 target_specific = get_target_table(vcpu->arch.target, false, &num);
1988 return kvm_handle_cp_32(vcpu,
1989 cp15_regs, ARRAY_SIZE(cp15_regs),
1990 target_specific, num);
1991}
1992
1993int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
1994{
1995 return kvm_handle_cp_64(vcpu,
1996 cp14_64_regs, ARRAY_SIZE(cp14_64_regs),
1997 NULL, 0);
1998}
1999
2000int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
2001{
2002 return kvm_handle_cp_32(vcpu,
2003 cp14_regs, ARRAY_SIZE(cp14_regs),
2004 NULL, 0);
2005}
2006
2007static int emulate_sys_reg(struct kvm_vcpu *vcpu,
2008 struct sys_reg_params *params)
2009{
2010 size_t num;
2011 const struct sys_reg_desc *table, *r;
2012
2013 table = get_target_table(vcpu->arch.target, true, &num);
2014
2015 /* Search target-specific then generic table. */
2016 r = find_reg(params, table, num);
2017 if (!r)
2018 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2019
2020 if (likely(r)) {
2021 perform_access(vcpu, params, r);
2022 } else {
2023 kvm_err("Unsupported guest sys_reg access at: %lx\n",
2024 *vcpu_pc(vcpu));
2025 print_sys_reg_instr(params);
2026 kvm_inject_undefined(vcpu);
2027 }
2028 return 1;
2029}
2030
2031static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
2032 const struct sys_reg_desc *table, size_t num)
2033{
2034 unsigned long i;
2035
2036 for (i = 0; i < num; i++)
2037 if (table[i].reset)
2038 table[i].reset(vcpu, &table[i]);
2039}
2040
2041/**
2042 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
2043 * @vcpu: The VCPU pointer
2044 * @run: The kvm_run struct
2045 */
2046int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
2047{
2048 struct sys_reg_params params;
2049 unsigned long esr = kvm_vcpu_get_hsr(vcpu);
2050 int Rt = kvm_vcpu_sys_get_rt(vcpu);
2051 int ret;
2052
2053 trace_kvm_handle_sys_reg(esr);
2054
2055 params.is_aarch32 = false;
2056 params.is_32bit = false;
2057 params.Op0 = (esr >> 20) & 3;
2058 params.Op1 = (esr >> 14) & 0x7;
2059 params.CRn = (esr >> 10) & 0xf;
2060 params.CRm = (esr >> 1) & 0xf;
2061 params.Op2 = (esr >> 17) & 0x7;
2062 params.regval = vcpu_get_reg(vcpu, Rt);
2063 params.is_write = !(esr & 1);
2064
2065 ret = emulate_sys_reg(vcpu, ¶ms);
2066
2067 if (!params.is_write)
2068 vcpu_set_reg(vcpu, Rt, params.regval);
2069 return ret;
2070}
2071
2072/******************************************************************************
2073 * Userspace API
2074 *****************************************************************************/
2075
2076static bool index_to_params(u64 id, struct sys_reg_params *params)
2077{
2078 switch (id & KVM_REG_SIZE_MASK) {
2079 case KVM_REG_SIZE_U64:
2080 /* Any unused index bits means it's not valid. */
2081 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
2082 | KVM_REG_ARM_COPROC_MASK
2083 | KVM_REG_ARM64_SYSREG_OP0_MASK
2084 | KVM_REG_ARM64_SYSREG_OP1_MASK
2085 | KVM_REG_ARM64_SYSREG_CRN_MASK
2086 | KVM_REG_ARM64_SYSREG_CRM_MASK
2087 | KVM_REG_ARM64_SYSREG_OP2_MASK))
2088 return false;
2089 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
2090 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
2091 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
2092 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
2093 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
2094 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
2095 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
2096 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
2097 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
2098 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
2099 return true;
2100 default:
2101 return false;
2102 }
2103}
2104
2105const struct sys_reg_desc *find_reg_by_id(u64 id,
2106 struct sys_reg_params *params,
2107 const struct sys_reg_desc table[],
2108 unsigned int num)
2109{
2110 if (!index_to_params(id, params))
2111 return NULL;
2112
2113 return find_reg(params, table, num);
2114}
2115
2116/* Decode an index value, and find the sys_reg_desc entry. */
2117static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
2118 u64 id)
2119{
2120 size_t num;
2121 const struct sys_reg_desc *table, *r;
2122 struct sys_reg_params params;
2123
2124 /* We only do sys_reg for now. */
2125 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
2126 return NULL;
2127
2128 table = get_target_table(vcpu->arch.target, true, &num);
2129 r = find_reg_by_id(id, ¶ms, table, num);
2130 if (!r)
2131 r = find_reg(¶ms, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2132
2133 /* Not saved in the sys_reg array and not otherwise accessible? */
2134 if (r && !(r->reg || r->get_user))
2135 r = NULL;
2136
2137 return r;
2138}
2139
2140/*
2141 * These are the invariant sys_reg registers: we let the guest see the
2142 * host versions of these, so they're part of the guest state.
2143 *
2144 * A future CPU may provide a mechanism to present different values to
2145 * the guest, or a future kvm may trap them.
2146 */
2147
2148#define FUNCTION_INVARIANT(reg) \
2149 static void get_##reg(struct kvm_vcpu *v, \
2150 const struct sys_reg_desc *r) \
2151 { \
2152 ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \
2153 }
2154
2155FUNCTION_INVARIANT(midr_el1)
2156FUNCTION_INVARIANT(ctr_el0)
2157FUNCTION_INVARIANT(revidr_el1)
2158FUNCTION_INVARIANT(clidr_el1)
2159FUNCTION_INVARIANT(aidr_el1)
2160
2161/* ->val is filled in by kvm_sys_reg_table_init() */
2162static struct sys_reg_desc invariant_sys_regs[] = {
2163 { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
2164 { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
2165 { SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
2166 { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
2167 { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
2168};
2169
2170static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
2171{
2172 if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
2173 return -EFAULT;
2174 return 0;
2175}
2176
2177static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
2178{
2179 if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
2180 return -EFAULT;
2181 return 0;
2182}
2183
2184static int get_invariant_sys_reg(u64 id, void __user *uaddr)
2185{
2186 struct sys_reg_params params;
2187 const struct sys_reg_desc *r;
2188
2189 r = find_reg_by_id(id, ¶ms, invariant_sys_regs,
2190 ARRAY_SIZE(invariant_sys_regs));
2191 if (!r)
2192 return -ENOENT;
2193
2194 return reg_to_user(uaddr, &r->val, id);
2195}
2196
2197static int set_invariant_sys_reg(u64 id, void __user *uaddr)
2198{
2199 struct sys_reg_params params;
2200 const struct sys_reg_desc *r;
2201 int err;
2202 u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
2203
2204 r = find_reg_by_id(id, ¶ms, invariant_sys_regs,
2205 ARRAY_SIZE(invariant_sys_regs));
2206 if (!r)
2207 return -ENOENT;
2208
2209 err = reg_from_user(&val, uaddr, id);
2210 if (err)
2211 return err;
2212
2213 /* This is what we mean by invariant: you can't change it. */
2214 if (r->val != val)
2215 return -EINVAL;
2216
2217 return 0;
2218}
2219
2220static bool is_valid_cache(u32 val)
2221{
2222 u32 level, ctype;
2223
2224 if (val >= CSSELR_MAX)
2225 return false;
2226
2227 /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */
2228 level = (val >> 1);
2229 ctype = (cache_levels >> (level * 3)) & 7;
2230
2231 switch (ctype) {
2232 case 0: /* No cache */
2233 return false;
2234 case 1: /* Instruction cache only */
2235 return (val & 1);
2236 case 2: /* Data cache only */
2237 case 4: /* Unified cache */
2238 return !(val & 1);
2239 case 3: /* Separate instruction and data caches */
2240 return true;
2241 default: /* Reserved: we can't know instruction or data. */
2242 return false;
2243 }
2244}
2245
2246static int demux_c15_get(u64 id, void __user *uaddr)
2247{
2248 u32 val;
2249 u32 __user *uval = uaddr;
2250
2251 /* Fail if we have unknown bits set. */
2252 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2253 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2254 return -ENOENT;
2255
2256 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2257 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2258 if (KVM_REG_SIZE(id) != 4)
2259 return -ENOENT;
2260 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2261 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2262 if (!is_valid_cache(val))
2263 return -ENOENT;
2264
2265 return put_user(get_ccsidr(val), uval);
2266 default:
2267 return -ENOENT;
2268 }
2269}
2270
2271static int demux_c15_set(u64 id, void __user *uaddr)
2272{
2273 u32 val, newval;
2274 u32 __user *uval = uaddr;
2275
2276 /* Fail if we have unknown bits set. */
2277 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2278 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2279 return -ENOENT;
2280
2281 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2282 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2283 if (KVM_REG_SIZE(id) != 4)
2284 return -ENOENT;
2285 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2286 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2287 if (!is_valid_cache(val))
2288 return -ENOENT;
2289
2290 if (get_user(newval, uval))
2291 return -EFAULT;
2292
2293 /* This is also invariant: you can't change it. */
2294 if (newval != get_ccsidr(val))
2295 return -EINVAL;
2296 return 0;
2297 default:
2298 return -ENOENT;
2299 }
2300}
2301
2302int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2303{
2304 const struct sys_reg_desc *r;
2305 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2306
2307 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2308 return demux_c15_get(reg->id, uaddr);
2309
2310 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2311 return -ENOENT;
2312
2313 r = index_to_sys_reg_desc(vcpu, reg->id);
2314 if (!r)
2315 return get_invariant_sys_reg(reg->id, uaddr);
2316
2317 if (r->get_user)
2318 return (r->get_user)(vcpu, r, reg, uaddr);
2319
2320 return reg_to_user(uaddr, &__vcpu_sys_reg(vcpu, r->reg), reg->id);
2321}
2322
2323int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2324{
2325 const struct sys_reg_desc *r;
2326 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2327
2328 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2329 return demux_c15_set(reg->id, uaddr);
2330
2331 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2332 return -ENOENT;
2333
2334 r = index_to_sys_reg_desc(vcpu, reg->id);
2335 if (!r)
2336 return set_invariant_sys_reg(reg->id, uaddr);
2337
2338 if (r->set_user)
2339 return (r->set_user)(vcpu, r, reg, uaddr);
2340
2341 return reg_from_user(&__vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
2342}
2343
2344static unsigned int num_demux_regs(void)
2345{
2346 unsigned int i, count = 0;
2347
2348 for (i = 0; i < CSSELR_MAX; i++)
2349 if (is_valid_cache(i))
2350 count++;
2351
2352 return count;
2353}
2354
2355static int write_demux_regids(u64 __user *uindices)
2356{
2357 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
2358 unsigned int i;
2359
2360 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
2361 for (i = 0; i < CSSELR_MAX; i++) {
2362 if (!is_valid_cache(i))
2363 continue;
2364 if (put_user(val | i, uindices))
2365 return -EFAULT;
2366 uindices++;
2367 }
2368 return 0;
2369}
2370
2371static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
2372{
2373 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
2374 KVM_REG_ARM64_SYSREG |
2375 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
2376 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
2377 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
2378 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
2379 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
2380}
2381
2382static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
2383{
2384 if (!*uind)
2385 return true;
2386
2387 if (put_user(sys_reg_to_index(reg), *uind))
2388 return false;
2389
2390 (*uind)++;
2391 return true;
2392}
2393
2394static int walk_one_sys_reg(const struct sys_reg_desc *rd,
2395 u64 __user **uind,
2396 unsigned int *total)
2397{
2398 /*
2399 * Ignore registers we trap but don't save,
2400 * and for which no custom user accessor is provided.
2401 */
2402 if (!(rd->reg || rd->get_user))
2403 return 0;
2404
2405 if (!copy_reg_to_user(rd, uind))
2406 return -EFAULT;
2407
2408 (*total)++;
2409 return 0;
2410}
2411
2412/* Assumed ordered tables, see kvm_sys_reg_table_init. */
2413static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
2414{
2415 const struct sys_reg_desc *i1, *i2, *end1, *end2;
2416 unsigned int total = 0;
2417 size_t num;
2418 int err;
2419
2420 /* We check for duplicates here, to allow arch-specific overrides. */
2421 i1 = get_target_table(vcpu->arch.target, true, &num);
2422 end1 = i1 + num;
2423 i2 = sys_reg_descs;
2424 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
2425
2426 BUG_ON(i1 == end1 || i2 == end2);
2427
2428 /* Walk carefully, as both tables may refer to the same register. */
2429 while (i1 || i2) {
2430 int cmp = cmp_sys_reg(i1, i2);
2431 /* target-specific overrides generic entry. */
2432 if (cmp <= 0)
2433 err = walk_one_sys_reg(i1, &uind, &total);
2434 else
2435 err = walk_one_sys_reg(i2, &uind, &total);
2436
2437 if (err)
2438 return err;
2439
2440 if (cmp <= 0 && ++i1 == end1)
2441 i1 = NULL;
2442 if (cmp >= 0 && ++i2 == end2)
2443 i2 = NULL;
2444 }
2445 return total;
2446}
2447
2448unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
2449{
2450 return ARRAY_SIZE(invariant_sys_regs)
2451 + num_demux_regs()
2452 + walk_sys_regs(vcpu, (u64 __user *)NULL);
2453}
2454
2455int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
2456{
2457 unsigned int i;
2458 int err;
2459
2460 /* Then give them all the invariant registers' indices. */
2461 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
2462 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
2463 return -EFAULT;
2464 uindices++;
2465 }
2466
2467 err = walk_sys_regs(vcpu, uindices);
2468 if (err < 0)
2469 return err;
2470 uindices += err;
2471
2472 return write_demux_regids(uindices);
2473}
2474
2475static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n)
2476{
2477 unsigned int i;
2478
2479 for (i = 1; i < n; i++) {
2480 if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2481 kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2482 return 1;
2483 }
2484 }
2485
2486 return 0;
2487}
2488
2489void kvm_sys_reg_table_init(void)
2490{
2491 unsigned int i;
2492 struct sys_reg_desc clidr;
2493
2494 /* Make sure tables are unique and in order. */
2495 BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs)));
2496 BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs)));
2497 BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs)));
2498 BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs)));
2499 BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs)));
2500 BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)));
2501
2502 /* We abuse the reset function to overwrite the table itself. */
2503 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
2504 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
2505
2506 /*
2507 * CLIDR format is awkward, so clean it up. See ARM B4.1.20:
2508 *
2509 * If software reads the Cache Type fields from Ctype1
2510 * upwards, once it has seen a value of 0b000, no caches
2511 * exist at further-out levels of the hierarchy. So, for
2512 * example, if Ctype3 is the first Cache Type field with a
2513 * value of 0b000, the values of Ctype4 to Ctype7 must be
2514 * ignored.
2515 */
2516 get_clidr_el1(NULL, &clidr); /* Ugly... */
2517 cache_levels = clidr.val;
2518 for (i = 0; i < 7; i++)
2519 if (((cache_levels >> (i*3)) & 7) == 0)
2520 break;
2521 /* Clear all higher bits. */
2522 cache_levels &= (1 << (i*3))-1;
2523}
2524
2525/**
2526 * kvm_reset_sys_regs - sets system registers to reset value
2527 * @vcpu: The VCPU pointer
2528 *
2529 * This function finds the right table above and sets the registers on the
2530 * virtual CPU struct to their architecturally defined reset values.
2531 */
2532void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
2533{
2534 size_t num;
2535 const struct sys_reg_desc *table;
2536
2537 /* Catch someone adding a register without putting in reset entry. */
2538 memset(&vcpu->arch.ctxt.sys_regs, 0x42, sizeof(vcpu->arch.ctxt.sys_regs));
2539
2540 /* Generic chip reset first (so target could override). */
2541 reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2542
2543 table = get_target_table(vcpu->arch.target, true, &num);
2544 reset_sys_reg_descs(vcpu, table, num);
2545
2546 for (num = 1; num < NR_SYS_REGS; num++)
2547 if (__vcpu_sys_reg(vcpu, num) == 0x4242424242424242)
2548 panic("Didn't reset __vcpu_sys_reg(%zi)", num);
2549}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/kvm/coproc.c:
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Authors: Rusty Russell <rusty@rustcorp.com.au>
9 * Christoffer Dall <c.dall@virtualopensystems.com>
10 */
11
12#include <linux/bitfield.h>
13#include <linux/bsearch.h>
14#include <linux/cacheinfo.h>
15#include <linux/kvm_host.h>
16#include <linux/mm.h>
17#include <linux/printk.h>
18#include <linux/uaccess.h>
19
20#include <asm/cacheflush.h>
21#include <asm/cputype.h>
22#include <asm/debug-monitors.h>
23#include <asm/esr.h>
24#include <asm/kvm_arm.h>
25#include <asm/kvm_emulate.h>
26#include <asm/kvm_hyp.h>
27#include <asm/kvm_mmu.h>
28#include <asm/kvm_nested.h>
29#include <asm/perf_event.h>
30#include <asm/sysreg.h>
31
32#include <trace/events/kvm.h>
33
34#include "sys_regs.h"
35
36#include "trace.h"
37
38/*
39 * For AArch32, we only take care of what is being trapped. Anything
40 * that has to do with init and userspace access has to go via the
41 * 64bit interface.
42 */
43
44static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
45static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
46 u64 val);
47
48static bool bad_trap(struct kvm_vcpu *vcpu,
49 struct sys_reg_params *params,
50 const struct sys_reg_desc *r,
51 const char *msg)
52{
53 WARN_ONCE(1, "Unexpected %s\n", msg);
54 print_sys_reg_instr(params);
55 kvm_inject_undefined(vcpu);
56 return false;
57}
58
59static bool read_from_write_only(struct kvm_vcpu *vcpu,
60 struct sys_reg_params *params,
61 const struct sys_reg_desc *r)
62{
63 return bad_trap(vcpu, params, r,
64 "sys_reg read to write-only register");
65}
66
67static bool write_to_read_only(struct kvm_vcpu *vcpu,
68 struct sys_reg_params *params,
69 const struct sys_reg_desc *r)
70{
71 return bad_trap(vcpu, params, r,
72 "sys_reg write to read-only register");
73}
74
75#define PURE_EL2_SYSREG(el2) \
76 case el2: { \
77 *el1r = el2; \
78 return true; \
79 }
80
81#define MAPPED_EL2_SYSREG(el2, el1, fn) \
82 case el2: { \
83 *xlate = fn; \
84 *el1r = el1; \
85 return true; \
86 }
87
88static bool get_el2_to_el1_mapping(unsigned int reg,
89 unsigned int *el1r, u64 (**xlate)(u64))
90{
91 switch (reg) {
92 PURE_EL2_SYSREG( VPIDR_EL2 );
93 PURE_EL2_SYSREG( VMPIDR_EL2 );
94 PURE_EL2_SYSREG( ACTLR_EL2 );
95 PURE_EL2_SYSREG( HCR_EL2 );
96 PURE_EL2_SYSREG( MDCR_EL2 );
97 PURE_EL2_SYSREG( HSTR_EL2 );
98 PURE_EL2_SYSREG( HACR_EL2 );
99 PURE_EL2_SYSREG( VTTBR_EL2 );
100 PURE_EL2_SYSREG( VTCR_EL2 );
101 PURE_EL2_SYSREG( RVBAR_EL2 );
102 PURE_EL2_SYSREG( TPIDR_EL2 );
103 PURE_EL2_SYSREG( HPFAR_EL2 );
104 PURE_EL2_SYSREG( CNTHCTL_EL2 );
105 MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1,
106 translate_sctlr_el2_to_sctlr_el1 );
107 MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1,
108 translate_cptr_el2_to_cpacr_el1 );
109 MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1,
110 translate_ttbr0_el2_to_ttbr0_el1 );
111 MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL );
112 MAPPED_EL2_SYSREG(TCR_EL2, TCR_EL1,
113 translate_tcr_el2_to_tcr_el1 );
114 MAPPED_EL2_SYSREG(VBAR_EL2, VBAR_EL1, NULL );
115 MAPPED_EL2_SYSREG(AFSR0_EL2, AFSR0_EL1, NULL );
116 MAPPED_EL2_SYSREG(AFSR1_EL2, AFSR1_EL1, NULL );
117 MAPPED_EL2_SYSREG(ESR_EL2, ESR_EL1, NULL );
118 MAPPED_EL2_SYSREG(FAR_EL2, FAR_EL1, NULL );
119 MAPPED_EL2_SYSREG(MAIR_EL2, MAIR_EL1, NULL );
120 MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );
121 MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );
122 MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
123 default:
124 return false;
125 }
126}
127
128u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
129{
130 u64 val = 0x8badf00d8badf00d;
131 u64 (*xlate)(u64) = NULL;
132 unsigned int el1r;
133
134 if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
135 goto memory_read;
136
137 if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
138 if (!is_hyp_ctxt(vcpu))
139 goto memory_read;
140
141 /*
142 * If this register does not have an EL1 counterpart,
143 * then read the stored EL2 version.
144 */
145 if (reg == el1r)
146 goto memory_read;
147
148 /*
149 * If we have a non-VHE guest and that the sysreg
150 * requires translation to be used at EL1, use the
151 * in-memory copy instead.
152 */
153 if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
154 goto memory_read;
155
156 /* Get the current version of the EL1 counterpart. */
157 WARN_ON(!__vcpu_read_sys_reg_from_cpu(el1r, &val));
158 return val;
159 }
160
161 /* EL1 register can't be on the CPU if the guest is in vEL2. */
162 if (unlikely(is_hyp_ctxt(vcpu)))
163 goto memory_read;
164
165 if (__vcpu_read_sys_reg_from_cpu(reg, &val))
166 return val;
167
168memory_read:
169 return __vcpu_sys_reg(vcpu, reg);
170}
171
172void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
173{
174 u64 (*xlate)(u64) = NULL;
175 unsigned int el1r;
176
177 if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
178 goto memory_write;
179
180 if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
181 if (!is_hyp_ctxt(vcpu))
182 goto memory_write;
183
184 /*
185 * Always store a copy of the write to memory to avoid having
186 * to reverse-translate virtual EL2 system registers for a
187 * non-VHE guest hypervisor.
188 */
189 __vcpu_sys_reg(vcpu, reg) = val;
190
191 /* No EL1 counterpart? We're done here.? */
192 if (reg == el1r)
193 return;
194
195 if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
196 val = xlate(val);
197
198 /* Redirect this to the EL1 version of the register. */
199 WARN_ON(!__vcpu_write_sys_reg_to_cpu(val, el1r));
200 return;
201 }
202
203 /* EL1 register can't be on the CPU if the guest is in vEL2. */
204 if (unlikely(is_hyp_ctxt(vcpu)))
205 goto memory_write;
206
207 if (__vcpu_write_sys_reg_to_cpu(val, reg))
208 return;
209
210memory_write:
211 __vcpu_sys_reg(vcpu, reg) = val;
212}
213
214/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
215#define CSSELR_MAX 14
216
217/*
218 * Returns the minimum line size for the selected cache, expressed as
219 * Log2(bytes).
220 */
221static u8 get_min_cache_line_size(bool icache)
222{
223 u64 ctr = read_sanitised_ftr_reg(SYS_CTR_EL0);
224 u8 field;
225
226 if (icache)
227 field = SYS_FIELD_GET(CTR_EL0, IminLine, ctr);
228 else
229 field = SYS_FIELD_GET(CTR_EL0, DminLine, ctr);
230
231 /*
232 * Cache line size is represented as Log2(words) in CTR_EL0.
233 * Log2(bytes) can be derived with the following:
234 *
235 * Log2(words) + 2 = Log2(bytes / 4) + 2
236 * = Log2(bytes) - 2 + 2
237 * = Log2(bytes)
238 */
239 return field + 2;
240}
241
242/* Which cache CCSIDR represents depends on CSSELR value. */
243static u32 get_ccsidr(struct kvm_vcpu *vcpu, u32 csselr)
244{
245 u8 line_size;
246
247 if (vcpu->arch.ccsidr)
248 return vcpu->arch.ccsidr[csselr];
249
250 line_size = get_min_cache_line_size(csselr & CSSELR_EL1_InD);
251
252 /*
253 * Fabricate a CCSIDR value as the overriding value does not exist.
254 * The real CCSIDR value will not be used as it can vary by the
255 * physical CPU which the vcpu currently resides in.
256 *
257 * The line size is determined with get_min_cache_line_size(), which
258 * should be valid for all CPUs even if they have different cache
259 * configuration.
260 *
261 * The associativity bits are cleared, meaning the geometry of all data
262 * and unified caches (which are guaranteed to be PIPT and thus
263 * non-aliasing) are 1 set and 1 way.
264 * Guests should not be doing cache operations by set/way at all, and
265 * for this reason, we trap them and attempt to infer the intent, so
266 * that we can flush the entire guest's address space at the appropriate
267 * time. The exposed geometry minimizes the number of the traps.
268 * [If guests should attempt to infer aliasing properties from the
269 * geometry (which is not permitted by the architecture), they would
270 * only do so for virtually indexed caches.]
271 *
272 * We don't check if the cache level exists as it is allowed to return
273 * an UNKNOWN value if not.
274 */
275 return SYS_FIELD_PREP(CCSIDR_EL1, LineSize, line_size - 4);
276}
277
278static int set_ccsidr(struct kvm_vcpu *vcpu, u32 csselr, u32 val)
279{
280 u8 line_size = FIELD_GET(CCSIDR_EL1_LineSize, val) + 4;
281 u32 *ccsidr = vcpu->arch.ccsidr;
282 u32 i;
283
284 if ((val & CCSIDR_EL1_RES0) ||
285 line_size < get_min_cache_line_size(csselr & CSSELR_EL1_InD))
286 return -EINVAL;
287
288 if (!ccsidr) {
289 if (val == get_ccsidr(vcpu, csselr))
290 return 0;
291
292 ccsidr = kmalloc_array(CSSELR_MAX, sizeof(u32), GFP_KERNEL_ACCOUNT);
293 if (!ccsidr)
294 return -ENOMEM;
295
296 for (i = 0; i < CSSELR_MAX; i++)
297 ccsidr[i] = get_ccsidr(vcpu, i);
298
299 vcpu->arch.ccsidr = ccsidr;
300 }
301
302 ccsidr[csselr] = val;
303
304 return 0;
305}
306
307static bool access_rw(struct kvm_vcpu *vcpu,
308 struct sys_reg_params *p,
309 const struct sys_reg_desc *r)
310{
311 if (p->is_write)
312 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
313 else
314 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
315
316 return true;
317}
318
319/*
320 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
321 */
322static bool access_dcsw(struct kvm_vcpu *vcpu,
323 struct sys_reg_params *p,
324 const struct sys_reg_desc *r)
325{
326 if (!p->is_write)
327 return read_from_write_only(vcpu, p, r);
328
329 /*
330 * Only track S/W ops if we don't have FWB. It still indicates
331 * that the guest is a bit broken (S/W operations should only
332 * be done by firmware, knowing that there is only a single
333 * CPU left in the system, and certainly not from non-secure
334 * software).
335 */
336 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
337 kvm_set_way_flush(vcpu);
338
339 return true;
340}
341
342static bool access_dcgsw(struct kvm_vcpu *vcpu,
343 struct sys_reg_params *p,
344 const struct sys_reg_desc *r)
345{
346 if (!kvm_has_mte(vcpu->kvm)) {
347 kvm_inject_undefined(vcpu);
348 return false;
349 }
350
351 /* Treat MTE S/W ops as we treat the classic ones: with contempt */
352 return access_dcsw(vcpu, p, r);
353}
354
355static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
356{
357 switch (r->aarch32_map) {
358 case AA32_LO:
359 *mask = GENMASK_ULL(31, 0);
360 *shift = 0;
361 break;
362 case AA32_HI:
363 *mask = GENMASK_ULL(63, 32);
364 *shift = 32;
365 break;
366 default:
367 *mask = GENMASK_ULL(63, 0);
368 *shift = 0;
369 break;
370 }
371}
372
373/*
374 * Generic accessor for VM registers. Only called as long as HCR_TVM
375 * is set. If the guest enables the MMU, we stop trapping the VM
376 * sys_regs and leave it in complete control of the caches.
377 */
378static bool access_vm_reg(struct kvm_vcpu *vcpu,
379 struct sys_reg_params *p,
380 const struct sys_reg_desc *r)
381{
382 bool was_enabled = vcpu_has_cache_enabled(vcpu);
383 u64 val, mask, shift;
384
385 BUG_ON(!p->is_write);
386
387 get_access_mask(r, &mask, &shift);
388
389 if (~mask) {
390 val = vcpu_read_sys_reg(vcpu, r->reg);
391 val &= ~mask;
392 } else {
393 val = 0;
394 }
395
396 val |= (p->regval & (mask >> shift)) << shift;
397 vcpu_write_sys_reg(vcpu, val, r->reg);
398
399 kvm_toggle_cache(vcpu, was_enabled);
400 return true;
401}
402
403static bool access_actlr(struct kvm_vcpu *vcpu,
404 struct sys_reg_params *p,
405 const struct sys_reg_desc *r)
406{
407 u64 mask, shift;
408
409 if (p->is_write)
410 return ignore_write(vcpu, p);
411
412 get_access_mask(r, &mask, &shift);
413 p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
414
415 return true;
416}
417
418/*
419 * Trap handler for the GICv3 SGI generation system register.
420 * Forward the request to the VGIC emulation.
421 * The cp15_64 code makes sure this automatically works
422 * for both AArch64 and AArch32 accesses.
423 */
424static bool access_gic_sgi(struct kvm_vcpu *vcpu,
425 struct sys_reg_params *p,
426 const struct sys_reg_desc *r)
427{
428 bool g1;
429
430 if (!p->is_write)
431 return read_from_write_only(vcpu, p, r);
432
433 /*
434 * In a system where GICD_CTLR.DS=1, a ICC_SGI0R_EL1 access generates
435 * Group0 SGIs only, while ICC_SGI1R_EL1 can generate either group,
436 * depending on the SGI configuration. ICC_ASGI1R_EL1 is effectively
437 * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
438 * group.
439 */
440 if (p->Op0 == 0) { /* AArch32 */
441 switch (p->Op1) {
442 default: /* Keep GCC quiet */
443 case 0: /* ICC_SGI1R */
444 g1 = true;
445 break;
446 case 1: /* ICC_ASGI1R */
447 case 2: /* ICC_SGI0R */
448 g1 = false;
449 break;
450 }
451 } else { /* AArch64 */
452 switch (p->Op2) {
453 default: /* Keep GCC quiet */
454 case 5: /* ICC_SGI1R_EL1 */
455 g1 = true;
456 break;
457 case 6: /* ICC_ASGI1R_EL1 */
458 case 7: /* ICC_SGI0R_EL1 */
459 g1 = false;
460 break;
461 }
462 }
463
464 vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
465
466 return true;
467}
468
469static bool access_gic_sre(struct kvm_vcpu *vcpu,
470 struct sys_reg_params *p,
471 const struct sys_reg_desc *r)
472{
473 if (p->is_write)
474 return ignore_write(vcpu, p);
475
476 p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
477 return true;
478}
479
480static bool trap_raz_wi(struct kvm_vcpu *vcpu,
481 struct sys_reg_params *p,
482 const struct sys_reg_desc *r)
483{
484 if (p->is_write)
485 return ignore_write(vcpu, p);
486 else
487 return read_zero(vcpu, p);
488}
489
490static bool trap_undef(struct kvm_vcpu *vcpu,
491 struct sys_reg_params *p,
492 const struct sys_reg_desc *r)
493{
494 kvm_inject_undefined(vcpu);
495 return false;
496}
497
498/*
499 * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
500 * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
501 * system, these registers should UNDEF. LORID_EL1 being a RO register, we
502 * treat it separately.
503 */
504static bool trap_loregion(struct kvm_vcpu *vcpu,
505 struct sys_reg_params *p,
506 const struct sys_reg_desc *r)
507{
508 u64 val = IDREG(vcpu->kvm, SYS_ID_AA64MMFR1_EL1);
509 u32 sr = reg_to_encoding(r);
510
511 if (!(val & (0xfUL << ID_AA64MMFR1_EL1_LO_SHIFT))) {
512 kvm_inject_undefined(vcpu);
513 return false;
514 }
515
516 if (p->is_write && sr == SYS_LORID_EL1)
517 return write_to_read_only(vcpu, p, r);
518
519 return trap_raz_wi(vcpu, p, r);
520}
521
522static bool trap_oslar_el1(struct kvm_vcpu *vcpu,
523 struct sys_reg_params *p,
524 const struct sys_reg_desc *r)
525{
526 u64 oslsr;
527
528 if (!p->is_write)
529 return read_from_write_only(vcpu, p, r);
530
531 /* Forward the OSLK bit to OSLSR */
532 oslsr = __vcpu_sys_reg(vcpu, OSLSR_EL1) & ~OSLSR_EL1_OSLK;
533 if (p->regval & OSLAR_EL1_OSLK)
534 oslsr |= OSLSR_EL1_OSLK;
535
536 __vcpu_sys_reg(vcpu, OSLSR_EL1) = oslsr;
537 return true;
538}
539
540static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
541 struct sys_reg_params *p,
542 const struct sys_reg_desc *r)
543{
544 if (p->is_write)
545 return write_to_read_only(vcpu, p, r);
546
547 p->regval = __vcpu_sys_reg(vcpu, r->reg);
548 return true;
549}
550
551static int set_oslsr_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
552 u64 val)
553{
554 /*
555 * The only modifiable bit is the OSLK bit. Refuse the write if
556 * userspace attempts to change any other bit in the register.
557 */
558 if ((val ^ rd->val) & ~OSLSR_EL1_OSLK)
559 return -EINVAL;
560
561 __vcpu_sys_reg(vcpu, rd->reg) = val;
562 return 0;
563}
564
565static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
566 struct sys_reg_params *p,
567 const struct sys_reg_desc *r)
568{
569 if (p->is_write) {
570 return ignore_write(vcpu, p);
571 } else {
572 p->regval = read_sysreg(dbgauthstatus_el1);
573 return true;
574 }
575}
576
577/*
578 * We want to avoid world-switching all the DBG registers all the
579 * time:
580 *
581 * - If we've touched any debug register, it is likely that we're
582 * going to touch more of them. It then makes sense to disable the
583 * traps and start doing the save/restore dance
584 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
585 * then mandatory to save/restore the registers, as the guest
586 * depends on them.
587 *
588 * For this, we use a DIRTY bit, indicating the guest has modified the
589 * debug registers, used as follow:
590 *
591 * On guest entry:
592 * - If the dirty bit is set (because we're coming back from trapping),
593 * disable the traps, save host registers, restore guest registers.
594 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
595 * set the dirty bit, disable the traps, save host registers,
596 * restore guest registers.
597 * - Otherwise, enable the traps
598 *
599 * On guest exit:
600 * - If the dirty bit is set, save guest registers, restore host
601 * registers and clear the dirty bit. This ensure that the host can
602 * now use the debug registers.
603 */
604static bool trap_debug_regs(struct kvm_vcpu *vcpu,
605 struct sys_reg_params *p,
606 const struct sys_reg_desc *r)
607{
608 access_rw(vcpu, p, r);
609 if (p->is_write)
610 vcpu_set_flag(vcpu, DEBUG_DIRTY);
611
612 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
613
614 return true;
615}
616
617/*
618 * reg_to_dbg/dbg_to_reg
619 *
620 * A 32 bit write to a debug register leave top bits alone
621 * A 32 bit read from a debug register only returns the bottom bits
622 *
623 * All writes will set the DEBUG_DIRTY flag to ensure the hyp code
624 * switches between host and guest values in future.
625 */
626static void reg_to_dbg(struct kvm_vcpu *vcpu,
627 struct sys_reg_params *p,
628 const struct sys_reg_desc *rd,
629 u64 *dbg_reg)
630{
631 u64 mask, shift, val;
632
633 get_access_mask(rd, &mask, &shift);
634
635 val = *dbg_reg;
636 val &= ~mask;
637 val |= (p->regval & (mask >> shift)) << shift;
638 *dbg_reg = val;
639
640 vcpu_set_flag(vcpu, DEBUG_DIRTY);
641}
642
643static void dbg_to_reg(struct kvm_vcpu *vcpu,
644 struct sys_reg_params *p,
645 const struct sys_reg_desc *rd,
646 u64 *dbg_reg)
647{
648 u64 mask, shift;
649
650 get_access_mask(rd, &mask, &shift);
651 p->regval = (*dbg_reg & mask) >> shift;
652}
653
654static bool trap_bvr(struct kvm_vcpu *vcpu,
655 struct sys_reg_params *p,
656 const struct sys_reg_desc *rd)
657{
658 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
659
660 if (p->is_write)
661 reg_to_dbg(vcpu, p, rd, dbg_reg);
662 else
663 dbg_to_reg(vcpu, p, rd, dbg_reg);
664
665 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
666
667 return true;
668}
669
670static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
671 u64 val)
672{
673 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = val;
674 return 0;
675}
676
677static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
678 u64 *val)
679{
680 *val = vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
681 return 0;
682}
683
684static u64 reset_bvr(struct kvm_vcpu *vcpu,
685 const struct sys_reg_desc *rd)
686{
687 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = rd->val;
688 return rd->val;
689}
690
691static bool trap_bcr(struct kvm_vcpu *vcpu,
692 struct sys_reg_params *p,
693 const struct sys_reg_desc *rd)
694{
695 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
696
697 if (p->is_write)
698 reg_to_dbg(vcpu, p, rd, dbg_reg);
699 else
700 dbg_to_reg(vcpu, p, rd, dbg_reg);
701
702 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
703
704 return true;
705}
706
707static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
708 u64 val)
709{
710 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = val;
711 return 0;
712}
713
714static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
715 u64 *val)
716{
717 *val = vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
718 return 0;
719}
720
721static u64 reset_bcr(struct kvm_vcpu *vcpu,
722 const struct sys_reg_desc *rd)
723{
724 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = rd->val;
725 return rd->val;
726}
727
728static bool trap_wvr(struct kvm_vcpu *vcpu,
729 struct sys_reg_params *p,
730 const struct sys_reg_desc *rd)
731{
732 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
733
734 if (p->is_write)
735 reg_to_dbg(vcpu, p, rd, dbg_reg);
736 else
737 dbg_to_reg(vcpu, p, rd, dbg_reg);
738
739 trace_trap_reg(__func__, rd->CRm, p->is_write,
740 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm]);
741
742 return true;
743}
744
745static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
746 u64 val)
747{
748 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = val;
749 return 0;
750}
751
752static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
753 u64 *val)
754{
755 *val = vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
756 return 0;
757}
758
759static u64 reset_wvr(struct kvm_vcpu *vcpu,
760 const struct sys_reg_desc *rd)
761{
762 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = rd->val;
763 return rd->val;
764}
765
766static bool trap_wcr(struct kvm_vcpu *vcpu,
767 struct sys_reg_params *p,
768 const struct sys_reg_desc *rd)
769{
770 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
771
772 if (p->is_write)
773 reg_to_dbg(vcpu, p, rd, dbg_reg);
774 else
775 dbg_to_reg(vcpu, p, rd, dbg_reg);
776
777 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
778
779 return true;
780}
781
782static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
783 u64 val)
784{
785 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = val;
786 return 0;
787}
788
789static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
790 u64 *val)
791{
792 *val = vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
793 return 0;
794}
795
796static u64 reset_wcr(struct kvm_vcpu *vcpu,
797 const struct sys_reg_desc *rd)
798{
799 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = rd->val;
800 return rd->val;
801}
802
803static u64 reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
804{
805 u64 amair = read_sysreg(amair_el1);
806 vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
807 return amair;
808}
809
810static u64 reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
811{
812 u64 actlr = read_sysreg(actlr_el1);
813 vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
814 return actlr;
815}
816
817static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
818{
819 u64 mpidr;
820
821 /*
822 * Map the vcpu_id into the first three affinity level fields of
823 * the MPIDR. We limit the number of VCPUs in level 0 due to a
824 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
825 * of the GICv3 to be able to address each CPU directly when
826 * sending IPIs.
827 */
828 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
829 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
830 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
831 mpidr |= (1ULL << 31);
832 vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1);
833
834 return mpidr;
835}
836
837static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
838 const struct sys_reg_desc *r)
839{
840 if (kvm_vcpu_has_pmu(vcpu))
841 return 0;
842
843 return REG_HIDDEN;
844}
845
846static u64 reset_pmu_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
847{
848 u64 mask = BIT(ARMV8_PMU_CYCLE_IDX);
849 u8 n = vcpu->kvm->arch.pmcr_n;
850
851 if (n)
852 mask |= GENMASK(n - 1, 0);
853
854 reset_unknown(vcpu, r);
855 __vcpu_sys_reg(vcpu, r->reg) &= mask;
856
857 return __vcpu_sys_reg(vcpu, r->reg);
858}
859
860static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
861{
862 reset_unknown(vcpu, r);
863 __vcpu_sys_reg(vcpu, r->reg) &= GENMASK(31, 0);
864
865 return __vcpu_sys_reg(vcpu, r->reg);
866}
867
868static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
869{
870 /* This thing will UNDEF, who cares about the reset value? */
871 if (!kvm_vcpu_has_pmu(vcpu))
872 return 0;
873
874 reset_unknown(vcpu, r);
875 __vcpu_sys_reg(vcpu, r->reg) &= kvm_pmu_evtyper_mask(vcpu->kvm);
876
877 return __vcpu_sys_reg(vcpu, r->reg);
878}
879
880static u64 reset_pmselr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
881{
882 reset_unknown(vcpu, r);
883 __vcpu_sys_reg(vcpu, r->reg) &= ARMV8_PMU_COUNTER_MASK;
884
885 return __vcpu_sys_reg(vcpu, r->reg);
886}
887
888static u64 reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
889{
890 u64 pmcr = 0;
891
892 if (!kvm_supports_32bit_el0())
893 pmcr |= ARMV8_PMU_PMCR_LC;
894
895 /*
896 * The value of PMCR.N field is included when the
897 * vCPU register is read via kvm_vcpu_read_pmcr().
898 */
899 __vcpu_sys_reg(vcpu, r->reg) = pmcr;
900
901 return __vcpu_sys_reg(vcpu, r->reg);
902}
903
904static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
905{
906 u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
907 bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
908
909 if (!enabled)
910 kvm_inject_undefined(vcpu);
911
912 return !enabled;
913}
914
915static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
916{
917 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
918}
919
920static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
921{
922 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
923}
924
925static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
926{
927 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
928}
929
930static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
931{
932 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
933}
934
935static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
936 const struct sys_reg_desc *r)
937{
938 u64 val;
939
940 if (pmu_access_el0_disabled(vcpu))
941 return false;
942
943 if (p->is_write) {
944 /*
945 * Only update writeable bits of PMCR (continuing into
946 * kvm_pmu_handle_pmcr() as well)
947 */
948 val = kvm_vcpu_read_pmcr(vcpu);
949 val &= ~ARMV8_PMU_PMCR_MASK;
950 val |= p->regval & ARMV8_PMU_PMCR_MASK;
951 if (!kvm_supports_32bit_el0())
952 val |= ARMV8_PMU_PMCR_LC;
953 kvm_pmu_handle_pmcr(vcpu, val);
954 } else {
955 /* PMCR.P & PMCR.C are RAZ */
956 val = kvm_vcpu_read_pmcr(vcpu)
957 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
958 p->regval = val;
959 }
960
961 return true;
962}
963
964static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
965 const struct sys_reg_desc *r)
966{
967 if (pmu_access_event_counter_el0_disabled(vcpu))
968 return false;
969
970 if (p->is_write)
971 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
972 else
973 /* return PMSELR.SEL field */
974 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
975 & ARMV8_PMU_COUNTER_MASK;
976
977 return true;
978}
979
980static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
981 const struct sys_reg_desc *r)
982{
983 u64 pmceid, mask, shift;
984
985 BUG_ON(p->is_write);
986
987 if (pmu_access_el0_disabled(vcpu))
988 return false;
989
990 get_access_mask(r, &mask, &shift);
991
992 pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
993 pmceid &= mask;
994 pmceid >>= shift;
995
996 p->regval = pmceid;
997
998 return true;
999}
1000
1001static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
1002{
1003 u64 pmcr, val;
1004
1005 pmcr = kvm_vcpu_read_pmcr(vcpu);
1006 val = FIELD_GET(ARMV8_PMU_PMCR_N, pmcr);
1007 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
1008 kvm_inject_undefined(vcpu);
1009 return false;
1010 }
1011
1012 return true;
1013}
1014
1015static int get_pmu_evcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1016 u64 *val)
1017{
1018 u64 idx;
1019
1020 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 0)
1021 /* PMCCNTR_EL0 */
1022 idx = ARMV8_PMU_CYCLE_IDX;
1023 else
1024 /* PMEVCNTRn_EL0 */
1025 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1026
1027 *val = kvm_pmu_get_counter_value(vcpu, idx);
1028 return 0;
1029}
1030
1031static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
1032 struct sys_reg_params *p,
1033 const struct sys_reg_desc *r)
1034{
1035 u64 idx = ~0UL;
1036
1037 if (r->CRn == 9 && r->CRm == 13) {
1038 if (r->Op2 == 2) {
1039 /* PMXEVCNTR_EL0 */
1040 if (pmu_access_event_counter_el0_disabled(vcpu))
1041 return false;
1042
1043 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
1044 & ARMV8_PMU_COUNTER_MASK;
1045 } else if (r->Op2 == 0) {
1046 /* PMCCNTR_EL0 */
1047 if (pmu_access_cycle_counter_el0_disabled(vcpu))
1048 return false;
1049
1050 idx = ARMV8_PMU_CYCLE_IDX;
1051 }
1052 } else if (r->CRn == 0 && r->CRm == 9) {
1053 /* PMCCNTR */
1054 if (pmu_access_event_counter_el0_disabled(vcpu))
1055 return false;
1056
1057 idx = ARMV8_PMU_CYCLE_IDX;
1058 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
1059 /* PMEVCNTRn_EL0 */
1060 if (pmu_access_event_counter_el0_disabled(vcpu))
1061 return false;
1062
1063 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1064 }
1065
1066 /* Catch any decoding mistake */
1067 WARN_ON(idx == ~0UL);
1068
1069 if (!pmu_counter_idx_valid(vcpu, idx))
1070 return false;
1071
1072 if (p->is_write) {
1073 if (pmu_access_el0_disabled(vcpu))
1074 return false;
1075
1076 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
1077 } else {
1078 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
1079 }
1080
1081 return true;
1082}
1083
1084static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1085 const struct sys_reg_desc *r)
1086{
1087 u64 idx, reg;
1088
1089 if (pmu_access_el0_disabled(vcpu))
1090 return false;
1091
1092 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
1093 /* PMXEVTYPER_EL0 */
1094 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
1095 reg = PMEVTYPER0_EL0 + idx;
1096 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
1097 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1098 if (idx == ARMV8_PMU_CYCLE_IDX)
1099 reg = PMCCFILTR_EL0;
1100 else
1101 /* PMEVTYPERn_EL0 */
1102 reg = PMEVTYPER0_EL0 + idx;
1103 } else {
1104 BUG();
1105 }
1106
1107 if (!pmu_counter_idx_valid(vcpu, idx))
1108 return false;
1109
1110 if (p->is_write) {
1111 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
1112 kvm_vcpu_pmu_restore_guest(vcpu);
1113 } else {
1114 p->regval = __vcpu_sys_reg(vcpu, reg);
1115 }
1116
1117 return true;
1118}
1119
1120static int set_pmreg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, u64 val)
1121{
1122 bool set;
1123
1124 val &= kvm_pmu_valid_counter_mask(vcpu);
1125
1126 switch (r->reg) {
1127 case PMOVSSET_EL0:
1128 /* CRm[1] being set indicates a SET register, and CLR otherwise */
1129 set = r->CRm & 2;
1130 break;
1131 default:
1132 /* Op2[0] being set indicates a SET register, and CLR otherwise */
1133 set = r->Op2 & 1;
1134 break;
1135 }
1136
1137 if (set)
1138 __vcpu_sys_reg(vcpu, r->reg) |= val;
1139 else
1140 __vcpu_sys_reg(vcpu, r->reg) &= ~val;
1141
1142 return 0;
1143}
1144
1145static int get_pmreg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, u64 *val)
1146{
1147 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1148
1149 *val = __vcpu_sys_reg(vcpu, r->reg) & mask;
1150 return 0;
1151}
1152
1153static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1154 const struct sys_reg_desc *r)
1155{
1156 u64 val, mask;
1157
1158 if (pmu_access_el0_disabled(vcpu))
1159 return false;
1160
1161 mask = kvm_pmu_valid_counter_mask(vcpu);
1162 if (p->is_write) {
1163 val = p->regval & mask;
1164 if (r->Op2 & 0x1) {
1165 /* accessing PMCNTENSET_EL0 */
1166 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
1167 kvm_pmu_enable_counter_mask(vcpu, val);
1168 kvm_vcpu_pmu_restore_guest(vcpu);
1169 } else {
1170 /* accessing PMCNTENCLR_EL0 */
1171 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
1172 kvm_pmu_disable_counter_mask(vcpu, val);
1173 }
1174 } else {
1175 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
1176 }
1177
1178 return true;
1179}
1180
1181static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1182 const struct sys_reg_desc *r)
1183{
1184 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1185
1186 if (check_pmu_access_disabled(vcpu, 0))
1187 return false;
1188
1189 if (p->is_write) {
1190 u64 val = p->regval & mask;
1191
1192 if (r->Op2 & 0x1)
1193 /* accessing PMINTENSET_EL1 */
1194 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
1195 else
1196 /* accessing PMINTENCLR_EL1 */
1197 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
1198 } else {
1199 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1);
1200 }
1201
1202 return true;
1203}
1204
1205static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1206 const struct sys_reg_desc *r)
1207{
1208 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1209
1210 if (pmu_access_el0_disabled(vcpu))
1211 return false;
1212
1213 if (p->is_write) {
1214 if (r->CRm & 0x2)
1215 /* accessing PMOVSSET_EL0 */
1216 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
1217 else
1218 /* accessing PMOVSCLR_EL0 */
1219 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
1220 } else {
1221 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);
1222 }
1223
1224 return true;
1225}
1226
1227static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1228 const struct sys_reg_desc *r)
1229{
1230 u64 mask;
1231
1232 if (!p->is_write)
1233 return read_from_write_only(vcpu, p, r);
1234
1235 if (pmu_write_swinc_el0_disabled(vcpu))
1236 return false;
1237
1238 mask = kvm_pmu_valid_counter_mask(vcpu);
1239 kvm_pmu_software_increment(vcpu, p->regval & mask);
1240 return true;
1241}
1242
1243static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1244 const struct sys_reg_desc *r)
1245{
1246 if (p->is_write) {
1247 if (!vcpu_mode_priv(vcpu)) {
1248 kvm_inject_undefined(vcpu);
1249 return false;
1250 }
1251
1252 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
1253 p->regval & ARMV8_PMU_USERENR_MASK;
1254 } else {
1255 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
1256 & ARMV8_PMU_USERENR_MASK;
1257 }
1258
1259 return true;
1260}
1261
1262static int get_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1263 u64 *val)
1264{
1265 *val = kvm_vcpu_read_pmcr(vcpu);
1266 return 0;
1267}
1268
1269static int set_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1270 u64 val)
1271{
1272 u8 new_n = FIELD_GET(ARMV8_PMU_PMCR_N, val);
1273 struct kvm *kvm = vcpu->kvm;
1274
1275 mutex_lock(&kvm->arch.config_lock);
1276
1277 /*
1278 * The vCPU can't have more counters than the PMU hardware
1279 * implements. Ignore this error to maintain compatibility
1280 * with the existing KVM behavior.
1281 */
1282 if (!kvm_vm_has_ran_once(kvm) &&
1283 new_n <= kvm_arm_pmu_get_max_counters(kvm))
1284 kvm->arch.pmcr_n = new_n;
1285
1286 mutex_unlock(&kvm->arch.config_lock);
1287
1288 /*
1289 * Ignore writes to RES0 bits, read only bits that are cleared on
1290 * vCPU reset, and writable bits that KVM doesn't support yet.
1291 * (i.e. only PMCR.N and bits [7:0] are mutable from userspace)
1292 * The LP bit is RES0 when FEAT_PMUv3p5 is not supported on the vCPU.
1293 * But, we leave the bit as it is here, as the vCPU's PMUver might
1294 * be changed later (NOTE: the bit will be cleared on first vCPU run
1295 * if necessary).
1296 */
1297 val &= ARMV8_PMU_PMCR_MASK;
1298
1299 /* The LC bit is RES1 when AArch32 is not supported */
1300 if (!kvm_supports_32bit_el0())
1301 val |= ARMV8_PMU_PMCR_LC;
1302
1303 __vcpu_sys_reg(vcpu, r->reg) = val;
1304 return 0;
1305}
1306
1307/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
1308#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
1309 { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
1310 trap_bvr, reset_bvr, 0, 0, get_bvr, set_bvr }, \
1311 { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
1312 trap_bcr, reset_bcr, 0, 0, get_bcr, set_bcr }, \
1313 { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
1314 trap_wvr, reset_wvr, 0, 0, get_wvr, set_wvr }, \
1315 { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
1316 trap_wcr, reset_wcr, 0, 0, get_wcr, set_wcr }
1317
1318#define PMU_SYS_REG(name) \
1319 SYS_DESC(SYS_##name), .reset = reset_pmu_reg, \
1320 .visibility = pmu_visibility
1321
1322/* Macro to expand the PMEVCNTRn_EL0 register */
1323#define PMU_PMEVCNTR_EL0(n) \
1324 { PMU_SYS_REG(PMEVCNTRn_EL0(n)), \
1325 .reset = reset_pmevcntr, .get_user = get_pmu_evcntr, \
1326 .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
1327
1328/* Macro to expand the PMEVTYPERn_EL0 register */
1329#define PMU_PMEVTYPER_EL0(n) \
1330 { PMU_SYS_REG(PMEVTYPERn_EL0(n)), \
1331 .reset = reset_pmevtyper, \
1332 .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
1333
1334static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1335 const struct sys_reg_desc *r)
1336{
1337 kvm_inject_undefined(vcpu);
1338
1339 return false;
1340}
1341
1342/* Macro to expand the AMU counter and type registers*/
1343#define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
1344#define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
1345#define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
1346#define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
1347
1348static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
1349 const struct sys_reg_desc *rd)
1350{
1351 return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
1352}
1353
1354/*
1355 * If we land here on a PtrAuth access, that is because we didn't
1356 * fixup the access on exit by allowing the PtrAuth sysregs. The only
1357 * way this happens is when the guest does not have PtrAuth support
1358 * enabled.
1359 */
1360#define __PTRAUTH_KEY(k) \
1361 { SYS_DESC(SYS_## k), undef_access, reset_unknown, k, \
1362 .visibility = ptrauth_visibility}
1363
1364#define PTRAUTH_KEY(k) \
1365 __PTRAUTH_KEY(k ## KEYLO_EL1), \
1366 __PTRAUTH_KEY(k ## KEYHI_EL1)
1367
1368static bool access_arch_timer(struct kvm_vcpu *vcpu,
1369 struct sys_reg_params *p,
1370 const struct sys_reg_desc *r)
1371{
1372 enum kvm_arch_timers tmr;
1373 enum kvm_arch_timer_regs treg;
1374 u64 reg = reg_to_encoding(r);
1375
1376 switch (reg) {
1377 case SYS_CNTP_TVAL_EL0:
1378 case SYS_AARCH32_CNTP_TVAL:
1379 tmr = TIMER_PTIMER;
1380 treg = TIMER_REG_TVAL;
1381 break;
1382 case SYS_CNTP_CTL_EL0:
1383 case SYS_AARCH32_CNTP_CTL:
1384 tmr = TIMER_PTIMER;
1385 treg = TIMER_REG_CTL;
1386 break;
1387 case SYS_CNTP_CVAL_EL0:
1388 case SYS_AARCH32_CNTP_CVAL:
1389 tmr = TIMER_PTIMER;
1390 treg = TIMER_REG_CVAL;
1391 break;
1392 case SYS_CNTPCT_EL0:
1393 case SYS_CNTPCTSS_EL0:
1394 case SYS_AARCH32_CNTPCT:
1395 tmr = TIMER_PTIMER;
1396 treg = TIMER_REG_CNT;
1397 break;
1398 default:
1399 print_sys_reg_msg(p, "%s", "Unhandled trapped timer register");
1400 kvm_inject_undefined(vcpu);
1401 return false;
1402 }
1403
1404 if (p->is_write)
1405 kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
1406 else
1407 p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
1408
1409 return true;
1410}
1411
1412static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp,
1413 s64 new, s64 cur)
1414{
1415 struct arm64_ftr_bits kvm_ftr = *ftrp;
1416
1417 /* Some features have different safe value type in KVM than host features */
1418 switch (id) {
1419 case SYS_ID_AA64DFR0_EL1:
1420 switch (kvm_ftr.shift) {
1421 case ID_AA64DFR0_EL1_PMUVer_SHIFT:
1422 kvm_ftr.type = FTR_LOWER_SAFE;
1423 break;
1424 case ID_AA64DFR0_EL1_DebugVer_SHIFT:
1425 kvm_ftr.type = FTR_LOWER_SAFE;
1426 break;
1427 }
1428 break;
1429 case SYS_ID_DFR0_EL1:
1430 if (kvm_ftr.shift == ID_DFR0_EL1_PerfMon_SHIFT)
1431 kvm_ftr.type = FTR_LOWER_SAFE;
1432 break;
1433 }
1434
1435 return arm64_ftr_safe_value(&kvm_ftr, new, cur);
1436}
1437
1438/*
1439 * arm64_check_features() - Check if a feature register value constitutes
1440 * a subset of features indicated by the idreg's KVM sanitised limit.
1441 *
1442 * This function will check if each feature field of @val is the "safe" value
1443 * against idreg's KVM sanitised limit return from reset() callback.
1444 * If a field value in @val is the same as the one in limit, it is always
1445 * considered the safe value regardless For register fields that are not in
1446 * writable, only the value in limit is considered the safe value.
1447 *
1448 * Return: 0 if all the fields are safe. Otherwise, return negative errno.
1449 */
1450static int arm64_check_features(struct kvm_vcpu *vcpu,
1451 const struct sys_reg_desc *rd,
1452 u64 val)
1453{
1454 const struct arm64_ftr_reg *ftr_reg;
1455 const struct arm64_ftr_bits *ftrp = NULL;
1456 u32 id = reg_to_encoding(rd);
1457 u64 writable_mask = rd->val;
1458 u64 limit = rd->reset(vcpu, rd);
1459 u64 mask = 0;
1460
1461 /*
1462 * Hidden and unallocated ID registers may not have a corresponding
1463 * struct arm64_ftr_reg. Of course, if the register is RAZ we know the
1464 * only safe value is 0.
1465 */
1466 if (sysreg_visible_as_raz(vcpu, rd))
1467 return val ? -E2BIG : 0;
1468
1469 ftr_reg = get_arm64_ftr_reg(id);
1470 if (!ftr_reg)
1471 return -EINVAL;
1472
1473 ftrp = ftr_reg->ftr_bits;
1474
1475 for (; ftrp && ftrp->width; ftrp++) {
1476 s64 f_val, f_lim, safe_val;
1477 u64 ftr_mask;
1478
1479 ftr_mask = arm64_ftr_mask(ftrp);
1480 if ((ftr_mask & writable_mask) != ftr_mask)
1481 continue;
1482
1483 f_val = arm64_ftr_value(ftrp, val);
1484 f_lim = arm64_ftr_value(ftrp, limit);
1485 mask |= ftr_mask;
1486
1487 if (f_val == f_lim)
1488 safe_val = f_val;
1489 else
1490 safe_val = kvm_arm64_ftr_safe_value(id, ftrp, f_val, f_lim);
1491
1492 if (safe_val != f_val)
1493 return -E2BIG;
1494 }
1495
1496 /* For fields that are not writable, values in limit are the safe values. */
1497 if ((val & ~mask) != (limit & ~mask))
1498 return -E2BIG;
1499
1500 return 0;
1501}
1502
1503static u8 pmuver_to_perfmon(u8 pmuver)
1504{
1505 switch (pmuver) {
1506 case ID_AA64DFR0_EL1_PMUVer_IMP:
1507 return ID_DFR0_EL1_PerfMon_PMUv3;
1508 case ID_AA64DFR0_EL1_PMUVer_IMP_DEF:
1509 return ID_DFR0_EL1_PerfMon_IMPDEF;
1510 default:
1511 /* Anything ARMv8.1+ and NI have the same value. For now. */
1512 return pmuver;
1513 }
1514}
1515
1516/* Read a sanitised cpufeature ID register by sys_reg_desc */
1517static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
1518 const struct sys_reg_desc *r)
1519{
1520 u32 id = reg_to_encoding(r);
1521 u64 val;
1522
1523 if (sysreg_visible_as_raz(vcpu, r))
1524 return 0;
1525
1526 val = read_sanitised_ftr_reg(id);
1527
1528 switch (id) {
1529 case SYS_ID_AA64PFR1_EL1:
1530 if (!kvm_has_mte(vcpu->kvm))
1531 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE);
1532
1533 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME);
1534 break;
1535 case SYS_ID_AA64ISAR1_EL1:
1536 if (!vcpu_has_ptrauth(vcpu))
1537 val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA) |
1538 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API) |
1539 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA) |
1540 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI));
1541 break;
1542 case SYS_ID_AA64ISAR2_EL1:
1543 if (!vcpu_has_ptrauth(vcpu))
1544 val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) |
1545 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
1546 if (!cpus_have_final_cap(ARM64_HAS_WFXT))
1547 val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT);
1548 break;
1549 case SYS_ID_AA64MMFR2_EL1:
1550 val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK;
1551 break;
1552 case SYS_ID_MMFR4_EL1:
1553 val &= ~ARM64_FEATURE_MASK(ID_MMFR4_EL1_CCIDX);
1554 break;
1555 }
1556
1557 return val;
1558}
1559
1560static u64 kvm_read_sanitised_id_reg(struct kvm_vcpu *vcpu,
1561 const struct sys_reg_desc *r)
1562{
1563 return __kvm_read_sanitised_id_reg(vcpu, r);
1564}
1565
1566static u64 read_id_reg(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
1567{
1568 return IDREG(vcpu->kvm, reg_to_encoding(r));
1569}
1570
1571/*
1572 * Return true if the register's (Op0, Op1, CRn, CRm, Op2) is
1573 * (3, 0, 0, crm, op2), where 1<=crm<8, 0<=op2<8.
1574 */
1575static inline bool is_id_reg(u32 id)
1576{
1577 return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 &&
1578 sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 &&
1579 sys_reg_CRm(id) < 8);
1580}
1581
1582static inline bool is_aa32_id_reg(u32 id)
1583{
1584 return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 &&
1585 sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 &&
1586 sys_reg_CRm(id) <= 3);
1587}
1588
1589static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1590 const struct sys_reg_desc *r)
1591{
1592 u32 id = reg_to_encoding(r);
1593
1594 switch (id) {
1595 case SYS_ID_AA64ZFR0_EL1:
1596 if (!vcpu_has_sve(vcpu))
1597 return REG_RAZ;
1598 break;
1599 }
1600
1601 return 0;
1602}
1603
1604static unsigned int aa32_id_visibility(const struct kvm_vcpu *vcpu,
1605 const struct sys_reg_desc *r)
1606{
1607 /*
1608 * AArch32 ID registers are UNKNOWN if AArch32 isn't implemented at any
1609 * EL. Promote to RAZ/WI in order to guarantee consistency between
1610 * systems.
1611 */
1612 if (!kvm_supports_32bit_el0())
1613 return REG_RAZ | REG_USER_WI;
1614
1615 return id_visibility(vcpu, r);
1616}
1617
1618static unsigned int raz_visibility(const struct kvm_vcpu *vcpu,
1619 const struct sys_reg_desc *r)
1620{
1621 return REG_RAZ;
1622}
1623
1624/* cpufeature ID register access trap handlers */
1625
1626static bool access_id_reg(struct kvm_vcpu *vcpu,
1627 struct sys_reg_params *p,
1628 const struct sys_reg_desc *r)
1629{
1630 if (p->is_write)
1631 return write_to_read_only(vcpu, p, r);
1632
1633 p->regval = read_id_reg(vcpu, r);
1634
1635 return true;
1636}
1637
1638/* Visibility overrides for SVE-specific control registers */
1639static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1640 const struct sys_reg_desc *rd)
1641{
1642 if (vcpu_has_sve(vcpu))
1643 return 0;
1644
1645 return REG_HIDDEN;
1646}
1647
1648static u64 read_sanitised_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1649 const struct sys_reg_desc *rd)
1650{
1651 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1652
1653 if (!vcpu_has_sve(vcpu))
1654 val &= ~ID_AA64PFR0_EL1_SVE_MASK;
1655
1656 /*
1657 * The default is to expose CSV2 == 1 if the HW isn't affected.
1658 * Although this is a per-CPU feature, we make it global because
1659 * asymmetric systems are just a nuisance.
1660 *
1661 * Userspace can override this as long as it doesn't promise
1662 * the impossible.
1663 */
1664 if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED) {
1665 val &= ~ID_AA64PFR0_EL1_CSV2_MASK;
1666 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV2, IMP);
1667 }
1668 if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED) {
1669 val &= ~ID_AA64PFR0_EL1_CSV3_MASK;
1670 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP);
1671 }
1672
1673 if (kvm_vgic_global_state.type == VGIC_V3) {
1674 val &= ~ID_AA64PFR0_EL1_GIC_MASK;
1675 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
1676 }
1677
1678 val &= ~ID_AA64PFR0_EL1_AMU_MASK;
1679
1680 return val;
1681}
1682
1683#define ID_REG_LIMIT_FIELD_ENUM(val, reg, field, limit) \
1684({ \
1685 u64 __f_val = FIELD_GET(reg##_##field##_MASK, val); \
1686 (val) &= ~reg##_##field##_MASK; \
1687 (val) |= FIELD_PREP(reg##_##field##_MASK, \
1688 min(__f_val, (u64)reg##_##field##_##limit)); \
1689 (val); \
1690})
1691
1692static u64 read_sanitised_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1693 const struct sys_reg_desc *rd)
1694{
1695 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1696
1697 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8);
1698
1699 /*
1700 * Only initialize the PMU version if the vCPU was configured with one.
1701 */
1702 val &= ~ID_AA64DFR0_EL1_PMUVer_MASK;
1703 if (kvm_vcpu_has_pmu(vcpu))
1704 val |= SYS_FIELD_PREP(ID_AA64DFR0_EL1, PMUVer,
1705 kvm_arm_pmu_get_pmuver_limit());
1706
1707 /* Hide SPE from guests */
1708 val &= ~ID_AA64DFR0_EL1_PMSVer_MASK;
1709
1710 return val;
1711}
1712
1713static int set_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1714 const struct sys_reg_desc *rd,
1715 u64 val)
1716{
1717 u8 debugver = SYS_FIELD_GET(ID_AA64DFR0_EL1, DebugVer, val);
1718 u8 pmuver = SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, val);
1719
1720 /*
1721 * Prior to commit 3d0dba5764b9 ("KVM: arm64: PMU: Move the
1722 * ID_AA64DFR0_EL1.PMUver limit to VM creation"), KVM erroneously
1723 * exposed an IMP_DEF PMU to userspace and the guest on systems w/
1724 * non-architectural PMUs. Of course, PMUv3 is the only game in town for
1725 * PMU virtualization, so the IMP_DEF value was rather user-hostile.
1726 *
1727 * At minimum, we're on the hook to allow values that were given to
1728 * userspace by KVM. Cover our tracks here and replace the IMP_DEF value
1729 * with a more sensible NI. The value of an ID register changing under
1730 * the nose of the guest is unfortunate, but is certainly no more
1731 * surprising than an ill-guided PMU driver poking at impdef system
1732 * registers that end in an UNDEF...
1733 */
1734 if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
1735 val &= ~ID_AA64DFR0_EL1_PMUVer_MASK;
1736
1737 /*
1738 * ID_AA64DFR0_EL1.DebugVer is one of those awkward fields with a
1739 * nonzero minimum safe value.
1740 */
1741 if (debugver < ID_AA64DFR0_EL1_DebugVer_IMP)
1742 return -EINVAL;
1743
1744 return set_id_reg(vcpu, rd, val);
1745}
1746
1747static u64 read_sanitised_id_dfr0_el1(struct kvm_vcpu *vcpu,
1748 const struct sys_reg_desc *rd)
1749{
1750 u8 perfmon = pmuver_to_perfmon(kvm_arm_pmu_get_pmuver_limit());
1751 u64 val = read_sanitised_ftr_reg(SYS_ID_DFR0_EL1);
1752
1753 val &= ~ID_DFR0_EL1_PerfMon_MASK;
1754 if (kvm_vcpu_has_pmu(vcpu))
1755 val |= SYS_FIELD_PREP(ID_DFR0_EL1, PerfMon, perfmon);
1756
1757 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_DFR0_EL1, CopDbg, Debugv8p8);
1758
1759 return val;
1760}
1761
1762static int set_id_dfr0_el1(struct kvm_vcpu *vcpu,
1763 const struct sys_reg_desc *rd,
1764 u64 val)
1765{
1766 u8 perfmon = SYS_FIELD_GET(ID_DFR0_EL1, PerfMon, val);
1767 u8 copdbg = SYS_FIELD_GET(ID_DFR0_EL1, CopDbg, val);
1768
1769 if (perfmon == ID_DFR0_EL1_PerfMon_IMPDEF) {
1770 val &= ~ID_DFR0_EL1_PerfMon_MASK;
1771 perfmon = 0;
1772 }
1773
1774 /*
1775 * Allow DFR0_EL1.PerfMon to be set from userspace as long as
1776 * it doesn't promise more than what the HW gives us on the
1777 * AArch64 side (as everything is emulated with that), and
1778 * that this is a PMUv3.
1779 */
1780 if (perfmon != 0 && perfmon < ID_DFR0_EL1_PerfMon_PMUv3)
1781 return -EINVAL;
1782
1783 if (copdbg < ID_DFR0_EL1_CopDbg_Armv8)
1784 return -EINVAL;
1785
1786 return set_id_reg(vcpu, rd, val);
1787}
1788
1789/*
1790 * cpufeature ID register user accessors
1791 *
1792 * For now, these registers are immutable for userspace, so no values
1793 * are stored, and for set_id_reg() we don't allow the effective value
1794 * to be changed.
1795 */
1796static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1797 u64 *val)
1798{
1799 /*
1800 * Avoid locking if the VM has already started, as the ID registers are
1801 * guaranteed to be invariant at that point.
1802 */
1803 if (kvm_vm_has_ran_once(vcpu->kvm)) {
1804 *val = read_id_reg(vcpu, rd);
1805 return 0;
1806 }
1807
1808 mutex_lock(&vcpu->kvm->arch.config_lock);
1809 *val = read_id_reg(vcpu, rd);
1810 mutex_unlock(&vcpu->kvm->arch.config_lock);
1811
1812 return 0;
1813}
1814
1815static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1816 u64 val)
1817{
1818 u32 id = reg_to_encoding(rd);
1819 int ret;
1820
1821 mutex_lock(&vcpu->kvm->arch.config_lock);
1822
1823 /*
1824 * Once the VM has started the ID registers are immutable. Reject any
1825 * write that does not match the final register value.
1826 */
1827 if (kvm_vm_has_ran_once(vcpu->kvm)) {
1828 if (val != read_id_reg(vcpu, rd))
1829 ret = -EBUSY;
1830 else
1831 ret = 0;
1832
1833 mutex_unlock(&vcpu->kvm->arch.config_lock);
1834 return ret;
1835 }
1836
1837 ret = arm64_check_features(vcpu, rd, val);
1838 if (!ret)
1839 IDREG(vcpu->kvm, id) = val;
1840
1841 mutex_unlock(&vcpu->kvm->arch.config_lock);
1842
1843 /*
1844 * arm64_check_features() returns -E2BIG to indicate the register's
1845 * feature set is a superset of the maximally-allowed register value.
1846 * While it would be nice to precisely describe this to userspace, the
1847 * existing UAPI for KVM_SET_ONE_REG has it that invalid register
1848 * writes return -EINVAL.
1849 */
1850 if (ret == -E2BIG)
1851 ret = -EINVAL;
1852 return ret;
1853}
1854
1855static int get_raz_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1856 u64 *val)
1857{
1858 *val = 0;
1859 return 0;
1860}
1861
1862static int set_wi_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1863 u64 val)
1864{
1865 return 0;
1866}
1867
1868static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1869 const struct sys_reg_desc *r)
1870{
1871 if (p->is_write)
1872 return write_to_read_only(vcpu, p, r);
1873
1874 p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1875 return true;
1876}
1877
1878static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1879 const struct sys_reg_desc *r)
1880{
1881 if (p->is_write)
1882 return write_to_read_only(vcpu, p, r);
1883
1884 p->regval = __vcpu_sys_reg(vcpu, r->reg);
1885 return true;
1886}
1887
1888/*
1889 * Fabricate a CLIDR_EL1 value instead of using the real value, which can vary
1890 * by the physical CPU which the vcpu currently resides in.
1891 */
1892static u64 reset_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
1893{
1894 u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
1895 u64 clidr;
1896 u8 loc;
1897
1898 if ((ctr_el0 & CTR_EL0_IDC)) {
1899 /*
1900 * Data cache clean to the PoU is not required so LoUU and LoUIS
1901 * will not be set and a unified cache, which will be marked as
1902 * LoC, will be added.
1903 *
1904 * If not DIC, let the unified cache L2 so that an instruction
1905 * cache can be added as L1 later.
1906 */
1907 loc = (ctr_el0 & CTR_EL0_DIC) ? 1 : 2;
1908 clidr = CACHE_TYPE_UNIFIED << CLIDR_CTYPE_SHIFT(loc);
1909 } else {
1910 /*
1911 * Data cache clean to the PoU is required so let L1 have a data
1912 * cache and mark it as LoUU and LoUIS. As L1 has a data cache,
1913 * it can be marked as LoC too.
1914 */
1915 loc = 1;
1916 clidr = 1 << CLIDR_LOUU_SHIFT;
1917 clidr |= 1 << CLIDR_LOUIS_SHIFT;
1918 clidr |= CACHE_TYPE_DATA << CLIDR_CTYPE_SHIFT(1);
1919 }
1920
1921 /*
1922 * Instruction cache invalidation to the PoU is required so let L1 have
1923 * an instruction cache. If L1 already has a data cache, it will be
1924 * CACHE_TYPE_SEPARATE.
1925 */
1926 if (!(ctr_el0 & CTR_EL0_DIC))
1927 clidr |= CACHE_TYPE_INST << CLIDR_CTYPE_SHIFT(1);
1928
1929 clidr |= loc << CLIDR_LOC_SHIFT;
1930
1931 /*
1932 * Add tag cache unified to data cache. Allocation tags and data are
1933 * unified in a cache line so that it looks valid even if there is only
1934 * one cache line.
1935 */
1936 if (kvm_has_mte(vcpu->kvm))
1937 clidr |= 2 << CLIDR_TTYPE_SHIFT(loc);
1938
1939 __vcpu_sys_reg(vcpu, r->reg) = clidr;
1940
1941 return __vcpu_sys_reg(vcpu, r->reg);
1942}
1943
1944static int set_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1945 u64 val)
1946{
1947 u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
1948 u64 idc = !CLIDR_LOC(val) || (!CLIDR_LOUIS(val) && !CLIDR_LOUU(val));
1949
1950 if ((val & CLIDR_EL1_RES0) || (!(ctr_el0 & CTR_EL0_IDC) && idc))
1951 return -EINVAL;
1952
1953 __vcpu_sys_reg(vcpu, rd->reg) = val;
1954
1955 return 0;
1956}
1957
1958static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1959 const struct sys_reg_desc *r)
1960{
1961 int reg = r->reg;
1962
1963 if (p->is_write)
1964 vcpu_write_sys_reg(vcpu, p->regval, reg);
1965 else
1966 p->regval = vcpu_read_sys_reg(vcpu, reg);
1967 return true;
1968}
1969
1970static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1971 const struct sys_reg_desc *r)
1972{
1973 u32 csselr;
1974
1975 if (p->is_write)
1976 return write_to_read_only(vcpu, p, r);
1977
1978 csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1979 csselr &= CSSELR_EL1_Level | CSSELR_EL1_InD;
1980 if (csselr < CSSELR_MAX)
1981 p->regval = get_ccsidr(vcpu, csselr);
1982
1983 return true;
1984}
1985
1986static unsigned int mte_visibility(const struct kvm_vcpu *vcpu,
1987 const struct sys_reg_desc *rd)
1988{
1989 if (kvm_has_mte(vcpu->kvm))
1990 return 0;
1991
1992 return REG_HIDDEN;
1993}
1994
1995#define MTE_REG(name) { \
1996 SYS_DESC(SYS_##name), \
1997 .access = undef_access, \
1998 .reset = reset_unknown, \
1999 .reg = name, \
2000 .visibility = mte_visibility, \
2001}
2002
2003static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
2004 const struct sys_reg_desc *rd)
2005{
2006 if (vcpu_has_nv(vcpu))
2007 return 0;
2008
2009 return REG_HIDDEN;
2010}
2011
2012static bool bad_vncr_trap(struct kvm_vcpu *vcpu,
2013 struct sys_reg_params *p,
2014 const struct sys_reg_desc *r)
2015{
2016 /*
2017 * We really shouldn't be here, and this is likely the result
2018 * of a misconfigured trap, as this register should target the
2019 * VNCR page, and nothing else.
2020 */
2021 return bad_trap(vcpu, p, r,
2022 "trap of VNCR-backed register");
2023}
2024
2025static bool bad_redir_trap(struct kvm_vcpu *vcpu,
2026 struct sys_reg_params *p,
2027 const struct sys_reg_desc *r)
2028{
2029 /*
2030 * We really shouldn't be here, and this is likely the result
2031 * of a misconfigured trap, as this register should target the
2032 * corresponding EL1, and nothing else.
2033 */
2034 return bad_trap(vcpu, p, r,
2035 "trap of EL2 register redirected to EL1");
2036}
2037
2038#define EL2_REG(name, acc, rst, v) { \
2039 SYS_DESC(SYS_##name), \
2040 .access = acc, \
2041 .reset = rst, \
2042 .reg = name, \
2043 .visibility = el2_visibility, \
2044 .val = v, \
2045}
2046
2047#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
2048#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
2049
2050/*
2051 * EL{0,1}2 registers are the EL2 view on an EL0 or EL1 register when
2052 * HCR_EL2.E2H==1, and only in the sysreg table for convenience of
2053 * handling traps. Given that, they are always hidden from userspace.
2054 */
2055static unsigned int hidden_user_visibility(const struct kvm_vcpu *vcpu,
2056 const struct sys_reg_desc *rd)
2057{
2058 return REG_HIDDEN_USER;
2059}
2060
2061#define EL12_REG(name, acc, rst, v) { \
2062 SYS_DESC(SYS_##name##_EL12), \
2063 .access = acc, \
2064 .reset = rst, \
2065 .reg = name##_EL1, \
2066 .val = v, \
2067 .visibility = hidden_user_visibility, \
2068}
2069
2070/*
2071 * Since reset() callback and field val are not used for idregs, they will be
2072 * used for specific purposes for idregs.
2073 * The reset() would return KVM sanitised register value. The value would be the
2074 * same as the host kernel sanitised value if there is no KVM sanitisation.
2075 * The val would be used as a mask indicating writable fields for the idreg.
2076 * Only bits with 1 are writable from userspace. This mask might not be
2077 * necessary in the future whenever all ID registers are enabled as writable
2078 * from userspace.
2079 */
2080
2081#define ID_DESC(name) \
2082 SYS_DESC(SYS_##name), \
2083 .access = access_id_reg, \
2084 .get_user = get_id_reg \
2085
2086/* sys_reg_desc initialiser for known cpufeature ID registers */
2087#define ID_SANITISED(name) { \
2088 ID_DESC(name), \
2089 .set_user = set_id_reg, \
2090 .visibility = id_visibility, \
2091 .reset = kvm_read_sanitised_id_reg, \
2092 .val = 0, \
2093}
2094
2095/* sys_reg_desc initialiser for known cpufeature ID registers */
2096#define AA32_ID_SANITISED(name) { \
2097 ID_DESC(name), \
2098 .set_user = set_id_reg, \
2099 .visibility = aa32_id_visibility, \
2100 .reset = kvm_read_sanitised_id_reg, \
2101 .val = 0, \
2102}
2103
2104/* sys_reg_desc initialiser for writable ID registers */
2105#define ID_WRITABLE(name, mask) { \
2106 ID_DESC(name), \
2107 .set_user = set_id_reg, \
2108 .visibility = id_visibility, \
2109 .reset = kvm_read_sanitised_id_reg, \
2110 .val = mask, \
2111}
2112
2113/*
2114 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
2115 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
2116 * (1 <= crm < 8, 0 <= Op2 < 8).
2117 */
2118#define ID_UNALLOCATED(crm, op2) { \
2119 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
2120 .access = access_id_reg, \
2121 .get_user = get_id_reg, \
2122 .set_user = set_id_reg, \
2123 .visibility = raz_visibility, \
2124 .reset = kvm_read_sanitised_id_reg, \
2125 .val = 0, \
2126}
2127
2128/*
2129 * sys_reg_desc initialiser for known ID registers that we hide from guests.
2130 * For now, these are exposed just like unallocated ID regs: they appear
2131 * RAZ for the guest.
2132 */
2133#define ID_HIDDEN(name) { \
2134 ID_DESC(name), \
2135 .set_user = set_id_reg, \
2136 .visibility = raz_visibility, \
2137 .reset = kvm_read_sanitised_id_reg, \
2138 .val = 0, \
2139}
2140
2141static bool access_sp_el1(struct kvm_vcpu *vcpu,
2142 struct sys_reg_params *p,
2143 const struct sys_reg_desc *r)
2144{
2145 if (p->is_write)
2146 __vcpu_sys_reg(vcpu, SP_EL1) = p->regval;
2147 else
2148 p->regval = __vcpu_sys_reg(vcpu, SP_EL1);
2149
2150 return true;
2151}
2152
2153static bool access_elr(struct kvm_vcpu *vcpu,
2154 struct sys_reg_params *p,
2155 const struct sys_reg_desc *r)
2156{
2157 if (p->is_write)
2158 vcpu_write_sys_reg(vcpu, p->regval, ELR_EL1);
2159 else
2160 p->regval = vcpu_read_sys_reg(vcpu, ELR_EL1);
2161
2162 return true;
2163}
2164
2165static bool access_spsr(struct kvm_vcpu *vcpu,
2166 struct sys_reg_params *p,
2167 const struct sys_reg_desc *r)
2168{
2169 if (p->is_write)
2170 __vcpu_sys_reg(vcpu, SPSR_EL1) = p->regval;
2171 else
2172 p->regval = __vcpu_sys_reg(vcpu, SPSR_EL1);
2173
2174 return true;
2175}
2176
2177/*
2178 * Architected system registers.
2179 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
2180 *
2181 * Debug handling: We do trap most, if not all debug related system
2182 * registers. The implementation is good enough to ensure that a guest
2183 * can use these with minimal performance degradation. The drawback is
2184 * that we don't implement any of the external debug architecture.
2185 * This should be revisited if we ever encounter a more demanding
2186 * guest...
2187 */
2188static const struct sys_reg_desc sys_reg_descs[] = {
2189 { SYS_DESC(SYS_DC_ISW), access_dcsw },
2190 { SYS_DESC(SYS_DC_IGSW), access_dcgsw },
2191 { SYS_DESC(SYS_DC_IGDSW), access_dcgsw },
2192 { SYS_DESC(SYS_DC_CSW), access_dcsw },
2193 { SYS_DESC(SYS_DC_CGSW), access_dcgsw },
2194 { SYS_DESC(SYS_DC_CGDSW), access_dcgsw },
2195 { SYS_DESC(SYS_DC_CISW), access_dcsw },
2196 { SYS_DESC(SYS_DC_CIGSW), access_dcgsw },
2197 { SYS_DESC(SYS_DC_CIGDSW), access_dcgsw },
2198
2199 DBG_BCR_BVR_WCR_WVR_EL1(0),
2200 DBG_BCR_BVR_WCR_WVR_EL1(1),
2201 { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
2202 { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
2203 DBG_BCR_BVR_WCR_WVR_EL1(2),
2204 DBG_BCR_BVR_WCR_WVR_EL1(3),
2205 DBG_BCR_BVR_WCR_WVR_EL1(4),
2206 DBG_BCR_BVR_WCR_WVR_EL1(5),
2207 DBG_BCR_BVR_WCR_WVR_EL1(6),
2208 DBG_BCR_BVR_WCR_WVR_EL1(7),
2209 DBG_BCR_BVR_WCR_WVR_EL1(8),
2210 DBG_BCR_BVR_WCR_WVR_EL1(9),
2211 DBG_BCR_BVR_WCR_WVR_EL1(10),
2212 DBG_BCR_BVR_WCR_WVR_EL1(11),
2213 DBG_BCR_BVR_WCR_WVR_EL1(12),
2214 DBG_BCR_BVR_WCR_WVR_EL1(13),
2215 DBG_BCR_BVR_WCR_WVR_EL1(14),
2216 DBG_BCR_BVR_WCR_WVR_EL1(15),
2217
2218 { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
2219 { SYS_DESC(SYS_OSLAR_EL1), trap_oslar_el1 },
2220 { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1, reset_val, OSLSR_EL1,
2221 OSLSR_EL1_OSLM_IMPLEMENTED, .set_user = set_oslsr_el1, },
2222 { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
2223 { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
2224 { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
2225 { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
2226 { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
2227
2228 { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
2229 { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
2230 // DBGDTR[TR]X_EL0 share the same encoding
2231 { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
2232
2233 { SYS_DESC(SYS_DBGVCR32_EL2), trap_undef, reset_val, DBGVCR32_EL2, 0 },
2234
2235 { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
2236
2237 /*
2238 * ID regs: all ID_SANITISED() entries here must have corresponding
2239 * entries in arm64_ftr_regs[].
2240 */
2241
2242 /* AArch64 mappings of the AArch32 ID registers */
2243 /* CRm=1 */
2244 AA32_ID_SANITISED(ID_PFR0_EL1),
2245 AA32_ID_SANITISED(ID_PFR1_EL1),
2246 { SYS_DESC(SYS_ID_DFR0_EL1),
2247 .access = access_id_reg,
2248 .get_user = get_id_reg,
2249 .set_user = set_id_dfr0_el1,
2250 .visibility = aa32_id_visibility,
2251 .reset = read_sanitised_id_dfr0_el1,
2252 .val = ID_DFR0_EL1_PerfMon_MASK |
2253 ID_DFR0_EL1_CopDbg_MASK, },
2254 ID_HIDDEN(ID_AFR0_EL1),
2255 AA32_ID_SANITISED(ID_MMFR0_EL1),
2256 AA32_ID_SANITISED(ID_MMFR1_EL1),
2257 AA32_ID_SANITISED(ID_MMFR2_EL1),
2258 AA32_ID_SANITISED(ID_MMFR3_EL1),
2259
2260 /* CRm=2 */
2261 AA32_ID_SANITISED(ID_ISAR0_EL1),
2262 AA32_ID_SANITISED(ID_ISAR1_EL1),
2263 AA32_ID_SANITISED(ID_ISAR2_EL1),
2264 AA32_ID_SANITISED(ID_ISAR3_EL1),
2265 AA32_ID_SANITISED(ID_ISAR4_EL1),
2266 AA32_ID_SANITISED(ID_ISAR5_EL1),
2267 AA32_ID_SANITISED(ID_MMFR4_EL1),
2268 AA32_ID_SANITISED(ID_ISAR6_EL1),
2269
2270 /* CRm=3 */
2271 AA32_ID_SANITISED(MVFR0_EL1),
2272 AA32_ID_SANITISED(MVFR1_EL1),
2273 AA32_ID_SANITISED(MVFR2_EL1),
2274 ID_UNALLOCATED(3,3),
2275 AA32_ID_SANITISED(ID_PFR2_EL1),
2276 ID_HIDDEN(ID_DFR1_EL1),
2277 AA32_ID_SANITISED(ID_MMFR5_EL1),
2278 ID_UNALLOCATED(3,7),
2279
2280 /* AArch64 ID registers */
2281 /* CRm=4 */
2282 { SYS_DESC(SYS_ID_AA64PFR0_EL1),
2283 .access = access_id_reg,
2284 .get_user = get_id_reg,
2285 .set_user = set_id_reg,
2286 .reset = read_sanitised_id_aa64pfr0_el1,
2287 .val = ~(ID_AA64PFR0_EL1_AMU |
2288 ID_AA64PFR0_EL1_MPAM |
2289 ID_AA64PFR0_EL1_SVE |
2290 ID_AA64PFR0_EL1_RAS |
2291 ID_AA64PFR0_EL1_GIC |
2292 ID_AA64PFR0_EL1_AdvSIMD |
2293 ID_AA64PFR0_EL1_FP), },
2294 ID_SANITISED(ID_AA64PFR1_EL1),
2295 ID_UNALLOCATED(4,2),
2296 ID_UNALLOCATED(4,3),
2297 ID_WRITABLE(ID_AA64ZFR0_EL1, ~ID_AA64ZFR0_EL1_RES0),
2298 ID_HIDDEN(ID_AA64SMFR0_EL1),
2299 ID_UNALLOCATED(4,6),
2300 ID_UNALLOCATED(4,7),
2301
2302 /* CRm=5 */
2303 { SYS_DESC(SYS_ID_AA64DFR0_EL1),
2304 .access = access_id_reg,
2305 .get_user = get_id_reg,
2306 .set_user = set_id_aa64dfr0_el1,
2307 .reset = read_sanitised_id_aa64dfr0_el1,
2308 .val = ID_AA64DFR0_EL1_PMUVer_MASK |
2309 ID_AA64DFR0_EL1_DebugVer_MASK, },
2310 ID_SANITISED(ID_AA64DFR1_EL1),
2311 ID_UNALLOCATED(5,2),
2312 ID_UNALLOCATED(5,3),
2313 ID_HIDDEN(ID_AA64AFR0_EL1),
2314 ID_HIDDEN(ID_AA64AFR1_EL1),
2315 ID_UNALLOCATED(5,6),
2316 ID_UNALLOCATED(5,7),
2317
2318 /* CRm=6 */
2319 ID_WRITABLE(ID_AA64ISAR0_EL1, ~ID_AA64ISAR0_EL1_RES0),
2320 ID_WRITABLE(ID_AA64ISAR1_EL1, ~(ID_AA64ISAR1_EL1_GPI |
2321 ID_AA64ISAR1_EL1_GPA |
2322 ID_AA64ISAR1_EL1_API |
2323 ID_AA64ISAR1_EL1_APA)),
2324 ID_WRITABLE(ID_AA64ISAR2_EL1, ~(ID_AA64ISAR2_EL1_RES0 |
2325 ID_AA64ISAR2_EL1_APA3 |
2326 ID_AA64ISAR2_EL1_GPA3)),
2327 ID_UNALLOCATED(6,3),
2328 ID_UNALLOCATED(6,4),
2329 ID_UNALLOCATED(6,5),
2330 ID_UNALLOCATED(6,6),
2331 ID_UNALLOCATED(6,7),
2332
2333 /* CRm=7 */
2334 ID_WRITABLE(ID_AA64MMFR0_EL1, ~(ID_AA64MMFR0_EL1_RES0 |
2335 ID_AA64MMFR0_EL1_TGRAN4_2 |
2336 ID_AA64MMFR0_EL1_TGRAN64_2 |
2337 ID_AA64MMFR0_EL1_TGRAN16_2)),
2338 ID_WRITABLE(ID_AA64MMFR1_EL1, ~(ID_AA64MMFR1_EL1_RES0 |
2339 ID_AA64MMFR1_EL1_HCX |
2340 ID_AA64MMFR1_EL1_XNX |
2341 ID_AA64MMFR1_EL1_TWED |
2342 ID_AA64MMFR1_EL1_XNX |
2343 ID_AA64MMFR1_EL1_VH |
2344 ID_AA64MMFR1_EL1_VMIDBits)),
2345 ID_WRITABLE(ID_AA64MMFR2_EL1, ~(ID_AA64MMFR2_EL1_RES0 |
2346 ID_AA64MMFR2_EL1_EVT |
2347 ID_AA64MMFR2_EL1_FWB |
2348 ID_AA64MMFR2_EL1_IDS |
2349 ID_AA64MMFR2_EL1_NV |
2350 ID_AA64MMFR2_EL1_CCIDX)),
2351 ID_SANITISED(ID_AA64MMFR3_EL1),
2352 ID_UNALLOCATED(7,4),
2353 ID_UNALLOCATED(7,5),
2354 ID_UNALLOCATED(7,6),
2355 ID_UNALLOCATED(7,7),
2356
2357 { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
2358 { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
2359 { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
2360
2361 MTE_REG(RGSR_EL1),
2362 MTE_REG(GCR_EL1),
2363
2364 { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
2365 { SYS_DESC(SYS_TRFCR_EL1), undef_access },
2366 { SYS_DESC(SYS_SMPRI_EL1), undef_access },
2367 { SYS_DESC(SYS_SMCR_EL1), undef_access },
2368 { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
2369 { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
2370 { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
2371 { SYS_DESC(SYS_TCR2_EL1), access_vm_reg, reset_val, TCR2_EL1, 0 },
2372
2373 PTRAUTH_KEY(APIA),
2374 PTRAUTH_KEY(APIB),
2375 PTRAUTH_KEY(APDA),
2376 PTRAUTH_KEY(APDB),
2377 PTRAUTH_KEY(APGA),
2378
2379 { SYS_DESC(SYS_SPSR_EL1), access_spsr},
2380 { SYS_DESC(SYS_ELR_EL1), access_elr},
2381
2382 { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
2383 { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
2384 { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
2385
2386 { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
2387 { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
2388 { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
2389 { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
2390 { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
2391 { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
2392 { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
2393 { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
2394
2395 MTE_REG(TFSR_EL1),
2396 MTE_REG(TFSRE0_EL1),
2397
2398 { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
2399 { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
2400
2401 { SYS_DESC(SYS_PMSCR_EL1), undef_access },
2402 { SYS_DESC(SYS_PMSNEVFR_EL1), undef_access },
2403 { SYS_DESC(SYS_PMSICR_EL1), undef_access },
2404 { SYS_DESC(SYS_PMSIRR_EL1), undef_access },
2405 { SYS_DESC(SYS_PMSFCR_EL1), undef_access },
2406 { SYS_DESC(SYS_PMSEVFR_EL1), undef_access },
2407 { SYS_DESC(SYS_PMSLATFR_EL1), undef_access },
2408 { SYS_DESC(SYS_PMSIDR_EL1), undef_access },
2409 { SYS_DESC(SYS_PMBLIMITR_EL1), undef_access },
2410 { SYS_DESC(SYS_PMBPTR_EL1), undef_access },
2411 { SYS_DESC(SYS_PMBSR_EL1), undef_access },
2412 /* PMBIDR_EL1 is not trapped */
2413
2414 { PMU_SYS_REG(PMINTENSET_EL1),
2415 .access = access_pminten, .reg = PMINTENSET_EL1,
2416 .get_user = get_pmreg, .set_user = set_pmreg },
2417 { PMU_SYS_REG(PMINTENCLR_EL1),
2418 .access = access_pminten, .reg = PMINTENSET_EL1,
2419 .get_user = get_pmreg, .set_user = set_pmreg },
2420 { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
2421
2422 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
2423 { SYS_DESC(SYS_PIRE0_EL1), NULL, reset_unknown, PIRE0_EL1 },
2424 { SYS_DESC(SYS_PIR_EL1), NULL, reset_unknown, PIR_EL1 },
2425 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
2426
2427 { SYS_DESC(SYS_LORSA_EL1), trap_loregion },
2428 { SYS_DESC(SYS_LOREA_EL1), trap_loregion },
2429 { SYS_DESC(SYS_LORN_EL1), trap_loregion },
2430 { SYS_DESC(SYS_LORC_EL1), trap_loregion },
2431 { SYS_DESC(SYS_LORID_EL1), trap_loregion },
2432
2433 { SYS_DESC(SYS_VBAR_EL1), access_rw, reset_val, VBAR_EL1, 0 },
2434 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
2435
2436 { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
2437 { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
2438 { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
2439 { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
2440 { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
2441 { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
2442 { SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
2443 { SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
2444 { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
2445 { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
2446 { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
2447 { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
2448
2449 { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
2450 { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
2451
2452 { SYS_DESC(SYS_ACCDATA_EL1), undef_access },
2453
2454 { SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
2455
2456 { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
2457
2458 { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
2459 { SYS_DESC(SYS_CLIDR_EL1), access_clidr, reset_clidr, CLIDR_EL1,
2460 .set_user = set_clidr },
2461 { SYS_DESC(SYS_CCSIDR2_EL1), undef_access },
2462 { SYS_DESC(SYS_SMIDR_EL1), undef_access },
2463 { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
2464 { SYS_DESC(SYS_CTR_EL0), access_ctr },
2465 { SYS_DESC(SYS_SVCR), undef_access },
2466
2467 { PMU_SYS_REG(PMCR_EL0), .access = access_pmcr, .reset = reset_pmcr,
2468 .reg = PMCR_EL0, .get_user = get_pmcr, .set_user = set_pmcr },
2469 { PMU_SYS_REG(PMCNTENSET_EL0),
2470 .access = access_pmcnten, .reg = PMCNTENSET_EL0,
2471 .get_user = get_pmreg, .set_user = set_pmreg },
2472 { PMU_SYS_REG(PMCNTENCLR_EL0),
2473 .access = access_pmcnten, .reg = PMCNTENSET_EL0,
2474 .get_user = get_pmreg, .set_user = set_pmreg },
2475 { PMU_SYS_REG(PMOVSCLR_EL0),
2476 .access = access_pmovs, .reg = PMOVSSET_EL0,
2477 .get_user = get_pmreg, .set_user = set_pmreg },
2478 /*
2479 * PM_SWINC_EL0 is exposed to userspace as RAZ/WI, as it was
2480 * previously (and pointlessly) advertised in the past...
2481 */
2482 { PMU_SYS_REG(PMSWINC_EL0),
2483 .get_user = get_raz_reg, .set_user = set_wi_reg,
2484 .access = access_pmswinc, .reset = NULL },
2485 { PMU_SYS_REG(PMSELR_EL0),
2486 .access = access_pmselr, .reset = reset_pmselr, .reg = PMSELR_EL0 },
2487 { PMU_SYS_REG(PMCEID0_EL0),
2488 .access = access_pmceid, .reset = NULL },
2489 { PMU_SYS_REG(PMCEID1_EL0),
2490 .access = access_pmceid, .reset = NULL },
2491 { PMU_SYS_REG(PMCCNTR_EL0),
2492 .access = access_pmu_evcntr, .reset = reset_unknown,
2493 .reg = PMCCNTR_EL0, .get_user = get_pmu_evcntr},
2494 { PMU_SYS_REG(PMXEVTYPER_EL0),
2495 .access = access_pmu_evtyper, .reset = NULL },
2496 { PMU_SYS_REG(PMXEVCNTR_EL0),
2497 .access = access_pmu_evcntr, .reset = NULL },
2498 /*
2499 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
2500 * in 32bit mode. Here we choose to reset it as zero for consistency.
2501 */
2502 { PMU_SYS_REG(PMUSERENR_EL0), .access = access_pmuserenr,
2503 .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
2504 { PMU_SYS_REG(PMOVSSET_EL0),
2505 .access = access_pmovs, .reg = PMOVSSET_EL0,
2506 .get_user = get_pmreg, .set_user = set_pmreg },
2507
2508 { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
2509 { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
2510 { SYS_DESC(SYS_TPIDR2_EL0), undef_access },
2511
2512 { SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
2513
2514 { SYS_DESC(SYS_AMCR_EL0), undef_access },
2515 { SYS_DESC(SYS_AMCFGR_EL0), undef_access },
2516 { SYS_DESC(SYS_AMCGCR_EL0), undef_access },
2517 { SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
2518 { SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
2519 { SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
2520 { SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
2521 { SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
2522 AMU_AMEVCNTR0_EL0(0),
2523 AMU_AMEVCNTR0_EL0(1),
2524 AMU_AMEVCNTR0_EL0(2),
2525 AMU_AMEVCNTR0_EL0(3),
2526 AMU_AMEVCNTR0_EL0(4),
2527 AMU_AMEVCNTR0_EL0(5),
2528 AMU_AMEVCNTR0_EL0(6),
2529 AMU_AMEVCNTR0_EL0(7),
2530 AMU_AMEVCNTR0_EL0(8),
2531 AMU_AMEVCNTR0_EL0(9),
2532 AMU_AMEVCNTR0_EL0(10),
2533 AMU_AMEVCNTR0_EL0(11),
2534 AMU_AMEVCNTR0_EL0(12),
2535 AMU_AMEVCNTR0_EL0(13),
2536 AMU_AMEVCNTR0_EL0(14),
2537 AMU_AMEVCNTR0_EL0(15),
2538 AMU_AMEVTYPER0_EL0(0),
2539 AMU_AMEVTYPER0_EL0(1),
2540 AMU_AMEVTYPER0_EL0(2),
2541 AMU_AMEVTYPER0_EL0(3),
2542 AMU_AMEVTYPER0_EL0(4),
2543 AMU_AMEVTYPER0_EL0(5),
2544 AMU_AMEVTYPER0_EL0(6),
2545 AMU_AMEVTYPER0_EL0(7),
2546 AMU_AMEVTYPER0_EL0(8),
2547 AMU_AMEVTYPER0_EL0(9),
2548 AMU_AMEVTYPER0_EL0(10),
2549 AMU_AMEVTYPER0_EL0(11),
2550 AMU_AMEVTYPER0_EL0(12),
2551 AMU_AMEVTYPER0_EL0(13),
2552 AMU_AMEVTYPER0_EL0(14),
2553 AMU_AMEVTYPER0_EL0(15),
2554 AMU_AMEVCNTR1_EL0(0),
2555 AMU_AMEVCNTR1_EL0(1),
2556 AMU_AMEVCNTR1_EL0(2),
2557 AMU_AMEVCNTR1_EL0(3),
2558 AMU_AMEVCNTR1_EL0(4),
2559 AMU_AMEVCNTR1_EL0(5),
2560 AMU_AMEVCNTR1_EL0(6),
2561 AMU_AMEVCNTR1_EL0(7),
2562 AMU_AMEVCNTR1_EL0(8),
2563 AMU_AMEVCNTR1_EL0(9),
2564 AMU_AMEVCNTR1_EL0(10),
2565 AMU_AMEVCNTR1_EL0(11),
2566 AMU_AMEVCNTR1_EL0(12),
2567 AMU_AMEVCNTR1_EL0(13),
2568 AMU_AMEVCNTR1_EL0(14),
2569 AMU_AMEVCNTR1_EL0(15),
2570 AMU_AMEVTYPER1_EL0(0),
2571 AMU_AMEVTYPER1_EL0(1),
2572 AMU_AMEVTYPER1_EL0(2),
2573 AMU_AMEVTYPER1_EL0(3),
2574 AMU_AMEVTYPER1_EL0(4),
2575 AMU_AMEVTYPER1_EL0(5),
2576 AMU_AMEVTYPER1_EL0(6),
2577 AMU_AMEVTYPER1_EL0(7),
2578 AMU_AMEVTYPER1_EL0(8),
2579 AMU_AMEVTYPER1_EL0(9),
2580 AMU_AMEVTYPER1_EL0(10),
2581 AMU_AMEVTYPER1_EL0(11),
2582 AMU_AMEVTYPER1_EL0(12),
2583 AMU_AMEVTYPER1_EL0(13),
2584 AMU_AMEVTYPER1_EL0(14),
2585 AMU_AMEVTYPER1_EL0(15),
2586
2587 { SYS_DESC(SYS_CNTPCT_EL0), access_arch_timer },
2588 { SYS_DESC(SYS_CNTPCTSS_EL0), access_arch_timer },
2589 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
2590 { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
2591 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
2592
2593 /* PMEVCNTRn_EL0 */
2594 PMU_PMEVCNTR_EL0(0),
2595 PMU_PMEVCNTR_EL0(1),
2596 PMU_PMEVCNTR_EL0(2),
2597 PMU_PMEVCNTR_EL0(3),
2598 PMU_PMEVCNTR_EL0(4),
2599 PMU_PMEVCNTR_EL0(5),
2600 PMU_PMEVCNTR_EL0(6),
2601 PMU_PMEVCNTR_EL0(7),
2602 PMU_PMEVCNTR_EL0(8),
2603 PMU_PMEVCNTR_EL0(9),
2604 PMU_PMEVCNTR_EL0(10),
2605 PMU_PMEVCNTR_EL0(11),
2606 PMU_PMEVCNTR_EL0(12),
2607 PMU_PMEVCNTR_EL0(13),
2608 PMU_PMEVCNTR_EL0(14),
2609 PMU_PMEVCNTR_EL0(15),
2610 PMU_PMEVCNTR_EL0(16),
2611 PMU_PMEVCNTR_EL0(17),
2612 PMU_PMEVCNTR_EL0(18),
2613 PMU_PMEVCNTR_EL0(19),
2614 PMU_PMEVCNTR_EL0(20),
2615 PMU_PMEVCNTR_EL0(21),
2616 PMU_PMEVCNTR_EL0(22),
2617 PMU_PMEVCNTR_EL0(23),
2618 PMU_PMEVCNTR_EL0(24),
2619 PMU_PMEVCNTR_EL0(25),
2620 PMU_PMEVCNTR_EL0(26),
2621 PMU_PMEVCNTR_EL0(27),
2622 PMU_PMEVCNTR_EL0(28),
2623 PMU_PMEVCNTR_EL0(29),
2624 PMU_PMEVCNTR_EL0(30),
2625 /* PMEVTYPERn_EL0 */
2626 PMU_PMEVTYPER_EL0(0),
2627 PMU_PMEVTYPER_EL0(1),
2628 PMU_PMEVTYPER_EL0(2),
2629 PMU_PMEVTYPER_EL0(3),
2630 PMU_PMEVTYPER_EL0(4),
2631 PMU_PMEVTYPER_EL0(5),
2632 PMU_PMEVTYPER_EL0(6),
2633 PMU_PMEVTYPER_EL0(7),
2634 PMU_PMEVTYPER_EL0(8),
2635 PMU_PMEVTYPER_EL0(9),
2636 PMU_PMEVTYPER_EL0(10),
2637 PMU_PMEVTYPER_EL0(11),
2638 PMU_PMEVTYPER_EL0(12),
2639 PMU_PMEVTYPER_EL0(13),
2640 PMU_PMEVTYPER_EL0(14),
2641 PMU_PMEVTYPER_EL0(15),
2642 PMU_PMEVTYPER_EL0(16),
2643 PMU_PMEVTYPER_EL0(17),
2644 PMU_PMEVTYPER_EL0(18),
2645 PMU_PMEVTYPER_EL0(19),
2646 PMU_PMEVTYPER_EL0(20),
2647 PMU_PMEVTYPER_EL0(21),
2648 PMU_PMEVTYPER_EL0(22),
2649 PMU_PMEVTYPER_EL0(23),
2650 PMU_PMEVTYPER_EL0(24),
2651 PMU_PMEVTYPER_EL0(25),
2652 PMU_PMEVTYPER_EL0(26),
2653 PMU_PMEVTYPER_EL0(27),
2654 PMU_PMEVTYPER_EL0(28),
2655 PMU_PMEVTYPER_EL0(29),
2656 PMU_PMEVTYPER_EL0(30),
2657 /*
2658 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
2659 * in 32bit mode. Here we choose to reset it as zero for consistency.
2660 */
2661 { PMU_SYS_REG(PMCCFILTR_EL0), .access = access_pmu_evtyper,
2662 .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
2663
2664 EL2_REG_VNCR(VPIDR_EL2, reset_unknown, 0),
2665 EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0),
2666 EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1),
2667 EL2_REG(ACTLR_EL2, access_rw, reset_val, 0),
2668 EL2_REG_VNCR(HCR_EL2, reset_val, 0),
2669 EL2_REG(MDCR_EL2, access_rw, reset_val, 0),
2670 EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1),
2671 EL2_REG_VNCR(HSTR_EL2, reset_val, 0),
2672 EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0),
2673 EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0),
2674 EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
2675 EL2_REG_VNCR(HACR_EL2, reset_val, 0),
2676
2677 EL2_REG_VNCR(HCRX_EL2, reset_val, 0),
2678
2679 EL2_REG(TTBR0_EL2, access_rw, reset_val, 0),
2680 EL2_REG(TTBR1_EL2, access_rw, reset_val, 0),
2681 EL2_REG(TCR_EL2, access_rw, reset_val, TCR_EL2_RES1),
2682 EL2_REG_VNCR(VTTBR_EL2, reset_val, 0),
2683 EL2_REG_VNCR(VTCR_EL2, reset_val, 0),
2684
2685 { SYS_DESC(SYS_DACR32_EL2), trap_undef, reset_unknown, DACR32_EL2 },
2686 EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
2687 EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0),
2688 EL2_REG_VNCR(HAFGRTR_EL2, reset_val, 0),
2689 EL2_REG_REDIR(SPSR_EL2, reset_val, 0),
2690 EL2_REG_REDIR(ELR_EL2, reset_val, 0),
2691 { SYS_DESC(SYS_SP_EL1), access_sp_el1},
2692
2693 /* AArch32 SPSR_* are RES0 if trapped from a NV guest */
2694 { SYS_DESC(SYS_SPSR_irq), .access = trap_raz_wi,
2695 .visibility = hidden_user_visibility },
2696 { SYS_DESC(SYS_SPSR_abt), .access = trap_raz_wi,
2697 .visibility = hidden_user_visibility },
2698 { SYS_DESC(SYS_SPSR_und), .access = trap_raz_wi,
2699 .visibility = hidden_user_visibility },
2700 { SYS_DESC(SYS_SPSR_fiq), .access = trap_raz_wi,
2701 .visibility = hidden_user_visibility },
2702
2703 { SYS_DESC(SYS_IFSR32_EL2), trap_undef, reset_unknown, IFSR32_EL2 },
2704 EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
2705 EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
2706 EL2_REG_REDIR(ESR_EL2, reset_val, 0),
2707 { SYS_DESC(SYS_FPEXC32_EL2), trap_undef, reset_val, FPEXC32_EL2, 0x700 },
2708
2709 EL2_REG_REDIR(FAR_EL2, reset_val, 0),
2710 EL2_REG(HPFAR_EL2, access_rw, reset_val, 0),
2711
2712 EL2_REG(MAIR_EL2, access_rw, reset_val, 0),
2713 EL2_REG(AMAIR_EL2, access_rw, reset_val, 0),
2714
2715 EL2_REG(VBAR_EL2, access_rw, reset_val, 0),
2716 EL2_REG(RVBAR_EL2, access_rw, reset_val, 0),
2717 { SYS_DESC(SYS_RMR_EL2), trap_undef },
2718
2719 EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0),
2720 EL2_REG(TPIDR_EL2, access_rw, reset_val, 0),
2721
2722 EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0),
2723 EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0),
2724
2725 EL12_REG(CNTKCTL, access_rw, reset_val, 0),
2726
2727 EL2_REG(SP_EL2, NULL, reset_unknown, 0),
2728};
2729
2730static const struct sys_reg_desc *first_idreg;
2731
2732static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
2733 struct sys_reg_params *p,
2734 const struct sys_reg_desc *r)
2735{
2736 if (p->is_write) {
2737 return ignore_write(vcpu, p);
2738 } else {
2739 u64 dfr = IDREG(vcpu->kvm, SYS_ID_AA64DFR0_EL1);
2740 u64 pfr = IDREG(vcpu->kvm, SYS_ID_AA64PFR0_EL1);
2741 u32 el3 = !!SYS_FIELD_GET(ID_AA64PFR0_EL1, EL3, pfr);
2742
2743 p->regval = ((SYS_FIELD_GET(ID_AA64DFR0_EL1, WRPs, dfr) << 28) |
2744 (SYS_FIELD_GET(ID_AA64DFR0_EL1, BRPs, dfr) << 24) |
2745 (SYS_FIELD_GET(ID_AA64DFR0_EL1, CTX_CMPs, dfr) << 20) |
2746 (SYS_FIELD_GET(ID_AA64DFR0_EL1, DebugVer, dfr) << 16) |
2747 (1 << 15) | (el3 << 14) | (el3 << 12));
2748 return true;
2749 }
2750}
2751
2752/*
2753 * AArch32 debug register mappings
2754 *
2755 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
2756 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
2757 *
2758 * None of the other registers share their location, so treat them as
2759 * if they were 64bit.
2760 */
2761#define DBG_BCR_BVR_WCR_WVR(n) \
2762 /* DBGBVRn */ \
2763 { AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
2764 /* DBGBCRn */ \
2765 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
2766 /* DBGWVRn */ \
2767 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
2768 /* DBGWCRn */ \
2769 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
2770
2771#define DBGBXVR(n) \
2772 { AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
2773
2774/*
2775 * Trapped cp14 registers. We generally ignore most of the external
2776 * debug, on the principle that they don't really make sense to a
2777 * guest. Revisit this one day, would this principle change.
2778 */
2779static const struct sys_reg_desc cp14_regs[] = {
2780 /* DBGDIDR */
2781 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
2782 /* DBGDTRRXext */
2783 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
2784
2785 DBG_BCR_BVR_WCR_WVR(0),
2786 /* DBGDSCRint */
2787 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
2788 DBG_BCR_BVR_WCR_WVR(1),
2789 /* DBGDCCINT */
2790 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
2791 /* DBGDSCRext */
2792 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
2793 DBG_BCR_BVR_WCR_WVR(2),
2794 /* DBGDTR[RT]Xint */
2795 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
2796 /* DBGDTR[RT]Xext */
2797 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
2798 DBG_BCR_BVR_WCR_WVR(3),
2799 DBG_BCR_BVR_WCR_WVR(4),
2800 DBG_BCR_BVR_WCR_WVR(5),
2801 /* DBGWFAR */
2802 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
2803 /* DBGOSECCR */
2804 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
2805 DBG_BCR_BVR_WCR_WVR(6),
2806 /* DBGVCR */
2807 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
2808 DBG_BCR_BVR_WCR_WVR(7),
2809 DBG_BCR_BVR_WCR_WVR(8),
2810 DBG_BCR_BVR_WCR_WVR(9),
2811 DBG_BCR_BVR_WCR_WVR(10),
2812 DBG_BCR_BVR_WCR_WVR(11),
2813 DBG_BCR_BVR_WCR_WVR(12),
2814 DBG_BCR_BVR_WCR_WVR(13),
2815 DBG_BCR_BVR_WCR_WVR(14),
2816 DBG_BCR_BVR_WCR_WVR(15),
2817
2818 /* DBGDRAR (32bit) */
2819 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
2820
2821 DBGBXVR(0),
2822 /* DBGOSLAR */
2823 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_oslar_el1 },
2824 DBGBXVR(1),
2825 /* DBGOSLSR */
2826 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1, NULL, OSLSR_EL1 },
2827 DBGBXVR(2),
2828 DBGBXVR(3),
2829 /* DBGOSDLR */
2830 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
2831 DBGBXVR(4),
2832 /* DBGPRCR */
2833 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
2834 DBGBXVR(5),
2835 DBGBXVR(6),
2836 DBGBXVR(7),
2837 DBGBXVR(8),
2838 DBGBXVR(9),
2839 DBGBXVR(10),
2840 DBGBXVR(11),
2841 DBGBXVR(12),
2842 DBGBXVR(13),
2843 DBGBXVR(14),
2844 DBGBXVR(15),
2845
2846 /* DBGDSAR (32bit) */
2847 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
2848
2849 /* DBGDEVID2 */
2850 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
2851 /* DBGDEVID1 */
2852 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
2853 /* DBGDEVID */
2854 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
2855 /* DBGCLAIMSET */
2856 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
2857 /* DBGCLAIMCLR */
2858 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
2859 /* DBGAUTHSTATUS */
2860 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
2861};
2862
2863/* Trapped cp14 64bit registers */
2864static const struct sys_reg_desc cp14_64_regs[] = {
2865 /* DBGDRAR (64bit) */
2866 { Op1( 0), CRm( 1), .access = trap_raz_wi },
2867
2868 /* DBGDSAR (64bit) */
2869 { Op1( 0), CRm( 2), .access = trap_raz_wi },
2870};
2871
2872#define CP15_PMU_SYS_REG(_map, _Op1, _CRn, _CRm, _Op2) \
2873 AA32(_map), \
2874 Op1(_Op1), CRn(_CRn), CRm(_CRm), Op2(_Op2), \
2875 .visibility = pmu_visibility
2876
2877/* Macro to expand the PMEVCNTRn register */
2878#define PMU_PMEVCNTR(n) \
2879 { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \
2880 (0b1000 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \
2881 .access = access_pmu_evcntr }
2882
2883/* Macro to expand the PMEVTYPERn register */
2884#define PMU_PMEVTYPER(n) \
2885 { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \
2886 (0b1100 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \
2887 .access = access_pmu_evtyper }
2888/*
2889 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
2890 * depending on the way they are accessed (as a 32bit or a 64bit
2891 * register).
2892 */
2893static const struct sys_reg_desc cp15_regs[] = {
2894 { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
2895 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
2896 /* ACTLR */
2897 { AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
2898 /* ACTLR2 */
2899 { AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
2900 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
2901 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
2902 /* TTBCR */
2903 { AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
2904 /* TTBCR2 */
2905 { AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
2906 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
2907 /* DFSR */
2908 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
2909 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
2910 /* ADFSR */
2911 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
2912 /* AIFSR */
2913 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
2914 /* DFAR */
2915 { AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
2916 /* IFAR */
2917 { AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
2918
2919 /*
2920 * DC{C,I,CI}SW operations:
2921 */
2922 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
2923 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
2924 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
2925
2926 /* PMU */
2927 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 0), .access = access_pmcr },
2928 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 1), .access = access_pmcnten },
2929 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 2), .access = access_pmcnten },
2930 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 3), .access = access_pmovs },
2931 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 4), .access = access_pmswinc },
2932 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 5), .access = access_pmselr },
2933 { CP15_PMU_SYS_REG(LO, 0, 9, 12, 6), .access = access_pmceid },
2934 { CP15_PMU_SYS_REG(LO, 0, 9, 12, 7), .access = access_pmceid },
2935 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 0), .access = access_pmu_evcntr },
2936 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 1), .access = access_pmu_evtyper },
2937 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 2), .access = access_pmu_evcntr },
2938 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 0), .access = access_pmuserenr },
2939 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 1), .access = access_pminten },
2940 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 2), .access = access_pminten },
2941 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 3), .access = access_pmovs },
2942 { CP15_PMU_SYS_REG(HI, 0, 9, 14, 4), .access = access_pmceid },
2943 { CP15_PMU_SYS_REG(HI, 0, 9, 14, 5), .access = access_pmceid },
2944 /* PMMIR */
2945 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = trap_raz_wi },
2946
2947 /* PRRR/MAIR0 */
2948 { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
2949 /* NMRR/MAIR1 */
2950 { AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
2951 /* AMAIR0 */
2952 { AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
2953 /* AMAIR1 */
2954 { AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
2955
2956 /* ICC_SRE */
2957 { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
2958
2959 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
2960
2961 /* Arch Tmers */
2962 { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
2963 { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
2964
2965 /* PMEVCNTRn */
2966 PMU_PMEVCNTR(0),
2967 PMU_PMEVCNTR(1),
2968 PMU_PMEVCNTR(2),
2969 PMU_PMEVCNTR(3),
2970 PMU_PMEVCNTR(4),
2971 PMU_PMEVCNTR(5),
2972 PMU_PMEVCNTR(6),
2973 PMU_PMEVCNTR(7),
2974 PMU_PMEVCNTR(8),
2975 PMU_PMEVCNTR(9),
2976 PMU_PMEVCNTR(10),
2977 PMU_PMEVCNTR(11),
2978 PMU_PMEVCNTR(12),
2979 PMU_PMEVCNTR(13),
2980 PMU_PMEVCNTR(14),
2981 PMU_PMEVCNTR(15),
2982 PMU_PMEVCNTR(16),
2983 PMU_PMEVCNTR(17),
2984 PMU_PMEVCNTR(18),
2985 PMU_PMEVCNTR(19),
2986 PMU_PMEVCNTR(20),
2987 PMU_PMEVCNTR(21),
2988 PMU_PMEVCNTR(22),
2989 PMU_PMEVCNTR(23),
2990 PMU_PMEVCNTR(24),
2991 PMU_PMEVCNTR(25),
2992 PMU_PMEVCNTR(26),
2993 PMU_PMEVCNTR(27),
2994 PMU_PMEVCNTR(28),
2995 PMU_PMEVCNTR(29),
2996 PMU_PMEVCNTR(30),
2997 /* PMEVTYPERn */
2998 PMU_PMEVTYPER(0),
2999 PMU_PMEVTYPER(1),
3000 PMU_PMEVTYPER(2),
3001 PMU_PMEVTYPER(3),
3002 PMU_PMEVTYPER(4),
3003 PMU_PMEVTYPER(5),
3004 PMU_PMEVTYPER(6),
3005 PMU_PMEVTYPER(7),
3006 PMU_PMEVTYPER(8),
3007 PMU_PMEVTYPER(9),
3008 PMU_PMEVTYPER(10),
3009 PMU_PMEVTYPER(11),
3010 PMU_PMEVTYPER(12),
3011 PMU_PMEVTYPER(13),
3012 PMU_PMEVTYPER(14),
3013 PMU_PMEVTYPER(15),
3014 PMU_PMEVTYPER(16),
3015 PMU_PMEVTYPER(17),
3016 PMU_PMEVTYPER(18),
3017 PMU_PMEVTYPER(19),
3018 PMU_PMEVTYPER(20),
3019 PMU_PMEVTYPER(21),
3020 PMU_PMEVTYPER(22),
3021 PMU_PMEVTYPER(23),
3022 PMU_PMEVTYPER(24),
3023 PMU_PMEVTYPER(25),
3024 PMU_PMEVTYPER(26),
3025 PMU_PMEVTYPER(27),
3026 PMU_PMEVTYPER(28),
3027 PMU_PMEVTYPER(29),
3028 PMU_PMEVTYPER(30),
3029 /* PMCCFILTR */
3030 { CP15_PMU_SYS_REG(DIRECT, 0, 14, 15, 7), .access = access_pmu_evtyper },
3031
3032 { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
3033 { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
3034
3035 /* CCSIDR2 */
3036 { Op1(1), CRn( 0), CRm( 0), Op2(2), undef_access },
3037
3038 { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
3039};
3040
3041static const struct sys_reg_desc cp15_64_regs[] = {
3042 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
3043 { CP15_PMU_SYS_REG(DIRECT, 0, 0, 9, 0), .access = access_pmu_evcntr },
3044 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
3045 { SYS_DESC(SYS_AARCH32_CNTPCT), access_arch_timer },
3046 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
3047 { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
3048 { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
3049 { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer },
3050 { SYS_DESC(SYS_AARCH32_CNTPCTSS), access_arch_timer },
3051};
3052
3053static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
3054 bool is_32)
3055{
3056 unsigned int i;
3057
3058 for (i = 0; i < n; i++) {
3059 if (!is_32 && table[i].reg && !table[i].reset) {
3060 kvm_err("sys_reg table %pS entry %d lacks reset\n", &table[i], i);
3061 return false;
3062 }
3063
3064 if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
3065 kvm_err("sys_reg table %pS entry %d out of order\n", &table[i - 1], i - 1);
3066 return false;
3067 }
3068 }
3069
3070 return true;
3071}
3072
3073int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
3074{
3075 kvm_inject_undefined(vcpu);
3076 return 1;
3077}
3078
3079static void perform_access(struct kvm_vcpu *vcpu,
3080 struct sys_reg_params *params,
3081 const struct sys_reg_desc *r)
3082{
3083 trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
3084
3085 /* Check for regs disabled by runtime config */
3086 if (sysreg_hidden(vcpu, r)) {
3087 kvm_inject_undefined(vcpu);
3088 return;
3089 }
3090
3091 /*
3092 * Not having an accessor means that we have configured a trap
3093 * that we don't know how to handle. This certainly qualifies
3094 * as a gross bug that should be fixed right away.
3095 */
3096 BUG_ON(!r->access);
3097
3098 /* Skip instruction if instructed so */
3099 if (likely(r->access(vcpu, params, r)))
3100 kvm_incr_pc(vcpu);
3101}
3102
3103/*
3104 * emulate_cp -- tries to match a sys_reg access in a handling table, and
3105 * call the corresponding trap handler.
3106 *
3107 * @params: pointer to the descriptor of the access
3108 * @table: array of trap descriptors
3109 * @num: size of the trap descriptor array
3110 *
3111 * Return true if the access has been handled, false if not.
3112 */
3113static bool emulate_cp(struct kvm_vcpu *vcpu,
3114 struct sys_reg_params *params,
3115 const struct sys_reg_desc *table,
3116 size_t num)
3117{
3118 const struct sys_reg_desc *r;
3119
3120 if (!table)
3121 return false; /* Not handled */
3122
3123 r = find_reg(params, table, num);
3124
3125 if (r) {
3126 perform_access(vcpu, params, r);
3127 return true;
3128 }
3129
3130 /* Not handled */
3131 return false;
3132}
3133
3134static void unhandled_cp_access(struct kvm_vcpu *vcpu,
3135 struct sys_reg_params *params)
3136{
3137 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
3138 int cp = -1;
3139
3140 switch (esr_ec) {
3141 case ESR_ELx_EC_CP15_32:
3142 case ESR_ELx_EC_CP15_64:
3143 cp = 15;
3144 break;
3145 case ESR_ELx_EC_CP14_MR:
3146 case ESR_ELx_EC_CP14_64:
3147 cp = 14;
3148 break;
3149 default:
3150 WARN_ON(1);
3151 }
3152
3153 print_sys_reg_msg(params,
3154 "Unsupported guest CP%d access at: %08lx [%08lx]\n",
3155 cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
3156 kvm_inject_undefined(vcpu);
3157}
3158
3159/**
3160 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
3161 * @vcpu: The VCPU pointer
3162 * @run: The kvm_run struct
3163 */
3164static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
3165 const struct sys_reg_desc *global,
3166 size_t nr_global)
3167{
3168 struct sys_reg_params params;
3169 u64 esr = kvm_vcpu_get_esr(vcpu);
3170 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3171 int Rt2 = (esr >> 10) & 0x1f;
3172
3173 params.CRm = (esr >> 1) & 0xf;
3174 params.is_write = ((esr & 1) == 0);
3175
3176 params.Op0 = 0;
3177 params.Op1 = (esr >> 16) & 0xf;
3178 params.Op2 = 0;
3179 params.CRn = 0;
3180
3181 /*
3182 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
3183 * backends between AArch32 and AArch64, we get away with it.
3184 */
3185 if (params.is_write) {
3186 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
3187 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
3188 }
3189
3190 /*
3191 * If the table contains a handler, handle the
3192 * potential register operation in the case of a read and return
3193 * with success.
3194 */
3195 if (emulate_cp(vcpu, ¶ms, global, nr_global)) {
3196 /* Split up the value between registers for the read side */
3197 if (!params.is_write) {
3198 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
3199 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
3200 }
3201
3202 return 1;
3203 }
3204
3205 unhandled_cp_access(vcpu, ¶ms);
3206 return 1;
3207}
3208
3209static bool emulate_sys_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *params);
3210
3211/*
3212 * The CP10 ID registers are architecturally mapped to AArch64 feature
3213 * registers. Abuse that fact so we can rely on the AArch64 handler for accesses
3214 * from AArch32.
3215 */
3216static bool kvm_esr_cp10_id_to_sys64(u64 esr, struct sys_reg_params *params)
3217{
3218 u8 reg_id = (esr >> 10) & 0xf;
3219 bool valid;
3220
3221 params->is_write = ((esr & 1) == 0);
3222 params->Op0 = 3;
3223 params->Op1 = 0;
3224 params->CRn = 0;
3225 params->CRm = 3;
3226
3227 /* CP10 ID registers are read-only */
3228 valid = !params->is_write;
3229
3230 switch (reg_id) {
3231 /* MVFR0 */
3232 case 0b0111:
3233 params->Op2 = 0;
3234 break;
3235 /* MVFR1 */
3236 case 0b0110:
3237 params->Op2 = 1;
3238 break;
3239 /* MVFR2 */
3240 case 0b0101:
3241 params->Op2 = 2;
3242 break;
3243 default:
3244 valid = false;
3245 }
3246
3247 if (valid)
3248 return true;
3249
3250 kvm_pr_unimpl("Unhandled cp10 register %s: %u\n",
3251 params->is_write ? "write" : "read", reg_id);
3252 return false;
3253}
3254
3255/**
3256 * kvm_handle_cp10_id() - Handles a VMRS trap on guest access to a 'Media and
3257 * VFP Register' from AArch32.
3258 * @vcpu: The vCPU pointer
3259 *
3260 * MVFR{0-2} are architecturally mapped to the AArch64 MVFR{0-2}_EL1 registers.
3261 * Work out the correct AArch64 system register encoding and reroute to the
3262 * AArch64 system register emulation.
3263 */
3264int kvm_handle_cp10_id(struct kvm_vcpu *vcpu)
3265{
3266 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3267 u64 esr = kvm_vcpu_get_esr(vcpu);
3268 struct sys_reg_params params;
3269
3270 /* UNDEF on any unhandled register access */
3271 if (!kvm_esr_cp10_id_to_sys64(esr, ¶ms)) {
3272 kvm_inject_undefined(vcpu);
3273 return 1;
3274 }
3275
3276 if (emulate_sys_reg(vcpu, ¶ms))
3277 vcpu_set_reg(vcpu, Rt, params.regval);
3278
3279 return 1;
3280}
3281
3282/**
3283 * kvm_emulate_cp15_id_reg() - Handles an MRC trap on a guest CP15 access where
3284 * CRn=0, which corresponds to the AArch32 feature
3285 * registers.
3286 * @vcpu: the vCPU pointer
3287 * @params: the system register access parameters.
3288 *
3289 * Our cp15 system register tables do not enumerate the AArch32 feature
3290 * registers. Conveniently, our AArch64 table does, and the AArch32 system
3291 * register encoding can be trivially remapped into the AArch64 for the feature
3292 * registers: Append op0=3, leaving op1, CRn, CRm, and op2 the same.
3293 *
3294 * According to DDI0487G.b G7.3.1, paragraph "Behavior of VMSAv8-32 32-bit
3295 * System registers with (coproc=0b1111, CRn==c0)", read accesses from this
3296 * range are either UNKNOWN or RES0. Rerouting remains architectural as we
3297 * treat undefined registers in this range as RAZ.
3298 */
3299static int kvm_emulate_cp15_id_reg(struct kvm_vcpu *vcpu,
3300 struct sys_reg_params *params)
3301{
3302 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3303
3304 /* Treat impossible writes to RO registers as UNDEFINED */
3305 if (params->is_write) {
3306 unhandled_cp_access(vcpu, params);
3307 return 1;
3308 }
3309
3310 params->Op0 = 3;
3311
3312 /*
3313 * All registers where CRm > 3 are known to be UNKNOWN/RAZ from AArch32.
3314 * Avoid conflicting with future expansion of AArch64 feature registers
3315 * and simply treat them as RAZ here.
3316 */
3317 if (params->CRm > 3)
3318 params->regval = 0;
3319 else if (!emulate_sys_reg(vcpu, params))
3320 return 1;
3321
3322 vcpu_set_reg(vcpu, Rt, params->regval);
3323 return 1;
3324}
3325
3326/**
3327 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
3328 * @vcpu: The VCPU pointer
3329 * @run: The kvm_run struct
3330 */
3331static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
3332 struct sys_reg_params *params,
3333 const struct sys_reg_desc *global,
3334 size_t nr_global)
3335{
3336 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3337
3338 params->regval = vcpu_get_reg(vcpu, Rt);
3339
3340 if (emulate_cp(vcpu, params, global, nr_global)) {
3341 if (!params->is_write)
3342 vcpu_set_reg(vcpu, Rt, params->regval);
3343 return 1;
3344 }
3345
3346 unhandled_cp_access(vcpu, params);
3347 return 1;
3348}
3349
3350int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
3351{
3352 return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
3353}
3354
3355int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
3356{
3357 struct sys_reg_params params;
3358
3359 params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu));
3360
3361 /*
3362 * Certain AArch32 ID registers are handled by rerouting to the AArch64
3363 * system register table. Registers in the ID range where CRm=0 are
3364 * excluded from this scheme as they do not trivially map into AArch64
3365 * system register encodings.
3366 */
3367 if (params.Op1 == 0 && params.CRn == 0 && params.CRm)
3368 return kvm_emulate_cp15_id_reg(vcpu, ¶ms);
3369
3370 return kvm_handle_cp_32(vcpu, ¶ms, cp15_regs, ARRAY_SIZE(cp15_regs));
3371}
3372
3373int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
3374{
3375 return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
3376}
3377
3378int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
3379{
3380 struct sys_reg_params params;
3381
3382 params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu));
3383
3384 return kvm_handle_cp_32(vcpu, ¶ms, cp14_regs, ARRAY_SIZE(cp14_regs));
3385}
3386
3387static bool is_imp_def_sys_reg(struct sys_reg_params *params)
3388{
3389 // See ARM DDI 0487E.a, section D12.3.2
3390 return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011;
3391}
3392
3393/**
3394 * emulate_sys_reg - Emulate a guest access to an AArch64 system register
3395 * @vcpu: The VCPU pointer
3396 * @params: Decoded system register parameters
3397 *
3398 * Return: true if the system register access was successful, false otherwise.
3399 */
3400static bool emulate_sys_reg(struct kvm_vcpu *vcpu,
3401 struct sys_reg_params *params)
3402{
3403 const struct sys_reg_desc *r;
3404
3405 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3406
3407 if (likely(r)) {
3408 perform_access(vcpu, params, r);
3409 return true;
3410 }
3411
3412 if (is_imp_def_sys_reg(params)) {
3413 kvm_inject_undefined(vcpu);
3414 } else {
3415 print_sys_reg_msg(params,
3416 "Unsupported guest sys_reg access at: %lx [%08lx]\n",
3417 *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
3418 kvm_inject_undefined(vcpu);
3419 }
3420 return false;
3421}
3422
3423static void kvm_reset_id_regs(struct kvm_vcpu *vcpu)
3424{
3425 const struct sys_reg_desc *idreg = first_idreg;
3426 u32 id = reg_to_encoding(idreg);
3427 struct kvm *kvm = vcpu->kvm;
3428
3429 if (test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags))
3430 return;
3431
3432 lockdep_assert_held(&kvm->arch.config_lock);
3433
3434 /* Initialize all idregs */
3435 while (is_id_reg(id)) {
3436 IDREG(kvm, id) = idreg->reset(vcpu, idreg);
3437
3438 idreg++;
3439 id = reg_to_encoding(idreg);
3440 }
3441
3442 set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags);
3443}
3444
3445/**
3446 * kvm_reset_sys_regs - sets system registers to reset value
3447 * @vcpu: The VCPU pointer
3448 *
3449 * This function finds the right table above and sets the registers on the
3450 * virtual CPU struct to their architecturally defined reset values.
3451 */
3452void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
3453{
3454 unsigned long i;
3455
3456 kvm_reset_id_regs(vcpu);
3457
3458 for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) {
3459 const struct sys_reg_desc *r = &sys_reg_descs[i];
3460
3461 if (is_id_reg(reg_to_encoding(r)))
3462 continue;
3463
3464 if (r->reset)
3465 r->reset(vcpu, r);
3466 }
3467}
3468
3469/**
3470 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
3471 * @vcpu: The VCPU pointer
3472 */
3473int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
3474{
3475 struct sys_reg_params params;
3476 unsigned long esr = kvm_vcpu_get_esr(vcpu);
3477 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3478
3479 trace_kvm_handle_sys_reg(esr);
3480
3481 if (__check_nv_sr_forward(vcpu))
3482 return 1;
3483
3484 params = esr_sys64_to_params(esr);
3485 params.regval = vcpu_get_reg(vcpu, Rt);
3486
3487 if (!emulate_sys_reg(vcpu, ¶ms))
3488 return 1;
3489
3490 if (!params.is_write)
3491 vcpu_set_reg(vcpu, Rt, params.regval);
3492 return 1;
3493}
3494
3495/******************************************************************************
3496 * Userspace API
3497 *****************************************************************************/
3498
3499static bool index_to_params(u64 id, struct sys_reg_params *params)
3500{
3501 switch (id & KVM_REG_SIZE_MASK) {
3502 case KVM_REG_SIZE_U64:
3503 /* Any unused index bits means it's not valid. */
3504 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
3505 | KVM_REG_ARM_COPROC_MASK
3506 | KVM_REG_ARM64_SYSREG_OP0_MASK
3507 | KVM_REG_ARM64_SYSREG_OP1_MASK
3508 | KVM_REG_ARM64_SYSREG_CRN_MASK
3509 | KVM_REG_ARM64_SYSREG_CRM_MASK
3510 | KVM_REG_ARM64_SYSREG_OP2_MASK))
3511 return false;
3512 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
3513 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
3514 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
3515 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
3516 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
3517 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
3518 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
3519 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
3520 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
3521 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
3522 return true;
3523 default:
3524 return false;
3525 }
3526}
3527
3528const struct sys_reg_desc *get_reg_by_id(u64 id,
3529 const struct sys_reg_desc table[],
3530 unsigned int num)
3531{
3532 struct sys_reg_params params;
3533
3534 if (!index_to_params(id, ¶ms))
3535 return NULL;
3536
3537 return find_reg(¶ms, table, num);
3538}
3539
3540/* Decode an index value, and find the sys_reg_desc entry. */
3541static const struct sys_reg_desc *
3542id_to_sys_reg_desc(struct kvm_vcpu *vcpu, u64 id,
3543 const struct sys_reg_desc table[], unsigned int num)
3544
3545{
3546 const struct sys_reg_desc *r;
3547
3548 /* We only do sys_reg for now. */
3549 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
3550 return NULL;
3551
3552 r = get_reg_by_id(id, table, num);
3553
3554 /* Not saved in the sys_reg array and not otherwise accessible? */
3555 if (r && (!(r->reg || r->get_user) || sysreg_hidden(vcpu, r)))
3556 r = NULL;
3557
3558 return r;
3559}
3560
3561/*
3562 * These are the invariant sys_reg registers: we let the guest see the
3563 * host versions of these, so they're part of the guest state.
3564 *
3565 * A future CPU may provide a mechanism to present different values to
3566 * the guest, or a future kvm may trap them.
3567 */
3568
3569#define FUNCTION_INVARIANT(reg) \
3570 static u64 get_##reg(struct kvm_vcpu *v, \
3571 const struct sys_reg_desc *r) \
3572 { \
3573 ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \
3574 return ((struct sys_reg_desc *)r)->val; \
3575 }
3576
3577FUNCTION_INVARIANT(midr_el1)
3578FUNCTION_INVARIANT(revidr_el1)
3579FUNCTION_INVARIANT(aidr_el1)
3580
3581static u64 get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
3582{
3583 ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
3584 return ((struct sys_reg_desc *)r)->val;
3585}
3586
3587/* ->val is filled in by kvm_sys_reg_table_init() */
3588static struct sys_reg_desc invariant_sys_regs[] __ro_after_init = {
3589 { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
3590 { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
3591 { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
3592 { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
3593};
3594
3595static int get_invariant_sys_reg(u64 id, u64 __user *uaddr)
3596{
3597 const struct sys_reg_desc *r;
3598
3599 r = get_reg_by_id(id, invariant_sys_regs,
3600 ARRAY_SIZE(invariant_sys_regs));
3601 if (!r)
3602 return -ENOENT;
3603
3604 return put_user(r->val, uaddr);
3605}
3606
3607static int set_invariant_sys_reg(u64 id, u64 __user *uaddr)
3608{
3609 const struct sys_reg_desc *r;
3610 u64 val;
3611
3612 r = get_reg_by_id(id, invariant_sys_regs,
3613 ARRAY_SIZE(invariant_sys_regs));
3614 if (!r)
3615 return -ENOENT;
3616
3617 if (get_user(val, uaddr))
3618 return -EFAULT;
3619
3620 /* This is what we mean by invariant: you can't change it. */
3621 if (r->val != val)
3622 return -EINVAL;
3623
3624 return 0;
3625}
3626
3627static int demux_c15_get(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
3628{
3629 u32 val;
3630 u32 __user *uval = uaddr;
3631
3632 /* Fail if we have unknown bits set. */
3633 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
3634 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
3635 return -ENOENT;
3636
3637 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
3638 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
3639 if (KVM_REG_SIZE(id) != 4)
3640 return -ENOENT;
3641 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
3642 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
3643 if (val >= CSSELR_MAX)
3644 return -ENOENT;
3645
3646 return put_user(get_ccsidr(vcpu, val), uval);
3647 default:
3648 return -ENOENT;
3649 }
3650}
3651
3652static int demux_c15_set(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
3653{
3654 u32 val, newval;
3655 u32 __user *uval = uaddr;
3656
3657 /* Fail if we have unknown bits set. */
3658 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
3659 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
3660 return -ENOENT;
3661
3662 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
3663 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
3664 if (KVM_REG_SIZE(id) != 4)
3665 return -ENOENT;
3666 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
3667 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
3668 if (val >= CSSELR_MAX)
3669 return -ENOENT;
3670
3671 if (get_user(newval, uval))
3672 return -EFAULT;
3673
3674 return set_ccsidr(vcpu, val, newval);
3675 default:
3676 return -ENOENT;
3677 }
3678}
3679
3680int kvm_sys_reg_get_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
3681 const struct sys_reg_desc table[], unsigned int num)
3682{
3683 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
3684 const struct sys_reg_desc *r;
3685 u64 val;
3686 int ret;
3687
3688 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
3689 if (!r || sysreg_hidden_user(vcpu, r))
3690 return -ENOENT;
3691
3692 if (r->get_user) {
3693 ret = (r->get_user)(vcpu, r, &val);
3694 } else {
3695 val = __vcpu_sys_reg(vcpu, r->reg);
3696 ret = 0;
3697 }
3698
3699 if (!ret)
3700 ret = put_user(val, uaddr);
3701
3702 return ret;
3703}
3704
3705int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
3706{
3707 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
3708 int err;
3709
3710 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
3711 return demux_c15_get(vcpu, reg->id, uaddr);
3712
3713 err = get_invariant_sys_reg(reg->id, uaddr);
3714 if (err != -ENOENT)
3715 return err;
3716
3717 return kvm_sys_reg_get_user(vcpu, reg,
3718 sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3719}
3720
3721int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
3722 const struct sys_reg_desc table[], unsigned int num)
3723{
3724 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
3725 const struct sys_reg_desc *r;
3726 u64 val;
3727 int ret;
3728
3729 if (get_user(val, uaddr))
3730 return -EFAULT;
3731
3732 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
3733 if (!r || sysreg_hidden_user(vcpu, r))
3734 return -ENOENT;
3735
3736 if (sysreg_user_write_ignore(vcpu, r))
3737 return 0;
3738
3739 if (r->set_user) {
3740 ret = (r->set_user)(vcpu, r, val);
3741 } else {
3742 __vcpu_sys_reg(vcpu, r->reg) = val;
3743 ret = 0;
3744 }
3745
3746 return ret;
3747}
3748
3749int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
3750{
3751 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
3752 int err;
3753
3754 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
3755 return demux_c15_set(vcpu, reg->id, uaddr);
3756
3757 err = set_invariant_sys_reg(reg->id, uaddr);
3758 if (err != -ENOENT)
3759 return err;
3760
3761 return kvm_sys_reg_set_user(vcpu, reg,
3762 sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3763}
3764
3765static unsigned int num_demux_regs(void)
3766{
3767 return CSSELR_MAX;
3768}
3769
3770static int write_demux_regids(u64 __user *uindices)
3771{
3772 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
3773 unsigned int i;
3774
3775 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
3776 for (i = 0; i < CSSELR_MAX; i++) {
3777 if (put_user(val | i, uindices))
3778 return -EFAULT;
3779 uindices++;
3780 }
3781 return 0;
3782}
3783
3784static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
3785{
3786 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
3787 KVM_REG_ARM64_SYSREG |
3788 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
3789 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
3790 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
3791 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
3792 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
3793}
3794
3795static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
3796{
3797 if (!*uind)
3798 return true;
3799
3800 if (put_user(sys_reg_to_index(reg), *uind))
3801 return false;
3802
3803 (*uind)++;
3804 return true;
3805}
3806
3807static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
3808 const struct sys_reg_desc *rd,
3809 u64 __user **uind,
3810 unsigned int *total)
3811{
3812 /*
3813 * Ignore registers we trap but don't save,
3814 * and for which no custom user accessor is provided.
3815 */
3816 if (!(rd->reg || rd->get_user))
3817 return 0;
3818
3819 if (sysreg_hidden_user(vcpu, rd))
3820 return 0;
3821
3822 if (!copy_reg_to_user(rd, uind))
3823 return -EFAULT;
3824
3825 (*total)++;
3826 return 0;
3827}
3828
3829/* Assumed ordered tables, see kvm_sys_reg_table_init. */
3830static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
3831{
3832 const struct sys_reg_desc *i2, *end2;
3833 unsigned int total = 0;
3834 int err;
3835
3836 i2 = sys_reg_descs;
3837 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
3838
3839 while (i2 != end2) {
3840 err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
3841 if (err)
3842 return err;
3843 }
3844 return total;
3845}
3846
3847unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
3848{
3849 return ARRAY_SIZE(invariant_sys_regs)
3850 + num_demux_regs()
3851 + walk_sys_regs(vcpu, (u64 __user *)NULL);
3852}
3853
3854int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
3855{
3856 unsigned int i;
3857 int err;
3858
3859 /* Then give them all the invariant registers' indices. */
3860 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
3861 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
3862 return -EFAULT;
3863 uindices++;
3864 }
3865
3866 err = walk_sys_regs(vcpu, uindices);
3867 if (err < 0)
3868 return err;
3869 uindices += err;
3870
3871 return write_demux_regids(uindices);
3872}
3873
3874#define KVM_ARM_FEATURE_ID_RANGE_INDEX(r) \
3875 KVM_ARM_FEATURE_ID_RANGE_IDX(sys_reg_Op0(r), \
3876 sys_reg_Op1(r), \
3877 sys_reg_CRn(r), \
3878 sys_reg_CRm(r), \
3879 sys_reg_Op2(r))
3880
3881static bool is_feature_id_reg(u32 encoding)
3882{
3883 return (sys_reg_Op0(encoding) == 3 &&
3884 (sys_reg_Op1(encoding) < 2 || sys_reg_Op1(encoding) == 3) &&
3885 sys_reg_CRn(encoding) == 0 &&
3886 sys_reg_CRm(encoding) <= 7);
3887}
3888
3889int kvm_vm_ioctl_get_reg_writable_masks(struct kvm *kvm, struct reg_mask_range *range)
3890{
3891 const void *zero_page = page_to_virt(ZERO_PAGE(0));
3892 u64 __user *masks = (u64 __user *)range->addr;
3893
3894 /* Only feature id range is supported, reserved[13] must be zero. */
3895 if (range->range ||
3896 memcmp(range->reserved, zero_page, sizeof(range->reserved)))
3897 return -EINVAL;
3898
3899 /* Wipe the whole thing first */
3900 if (clear_user(masks, KVM_ARM_FEATURE_ID_RANGE_SIZE * sizeof(__u64)))
3901 return -EFAULT;
3902
3903 for (int i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) {
3904 const struct sys_reg_desc *reg = &sys_reg_descs[i];
3905 u32 encoding = reg_to_encoding(reg);
3906 u64 val;
3907
3908 if (!is_feature_id_reg(encoding) || !reg->set_user)
3909 continue;
3910
3911 /*
3912 * For ID registers, we return the writable mask. Other feature
3913 * registers return a full 64bit mask. That's not necessary
3914 * compliant with a given revision of the architecture, but the
3915 * RES0/RES1 definitions allow us to do that.
3916 */
3917 if (is_id_reg(encoding)) {
3918 if (!reg->val ||
3919 (is_aa32_id_reg(encoding) && !kvm_supports_32bit_el0()))
3920 continue;
3921 val = reg->val;
3922 } else {
3923 val = ~0UL;
3924 }
3925
3926 if (put_user(val, (masks + KVM_ARM_FEATURE_ID_RANGE_INDEX(encoding))))
3927 return -EFAULT;
3928 }
3929
3930 return 0;
3931}
3932
3933int __init kvm_sys_reg_table_init(void)
3934{
3935 struct sys_reg_params params;
3936 bool valid = true;
3937 unsigned int i;
3938
3939 /* Make sure tables are unique and in order. */
3940 valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false);
3941 valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true);
3942 valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true);
3943 valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true);
3944 valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true);
3945 valid &= check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false);
3946
3947 if (!valid)
3948 return -EINVAL;
3949
3950 /* We abuse the reset function to overwrite the table itself. */
3951 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
3952 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
3953
3954 /* Find the first idreg (SYS_ID_PFR0_EL1) in sys_reg_descs. */
3955 params = encoding_to_params(SYS_ID_PFR0_EL1);
3956 first_idreg = find_reg(¶ms, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3957 if (!first_idreg)
3958 return -EINVAL;
3959
3960 if (kvm_get_mode() == KVM_MODE_NV)
3961 return populate_nv_trap_config();
3962
3963 return 0;
3964}