Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/kvm/guest.c:
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
9 */
10
11#include <linux/bits.h>
12#include <linux/errno.h>
13#include <linux/err.h>
14#include <linux/nospec.h>
15#include <linux/kvm_host.h>
16#include <linux/module.h>
17#include <linux/stddef.h>
18#include <linux/string.h>
19#include <linux/vmalloc.h>
20#include <linux/fs.h>
21#include <kvm/arm_psci.h>
22#include <asm/cputype.h>
23#include <linux/uaccess.h>
24#include <asm/fpsimd.h>
25#include <asm/kvm.h>
26#include <asm/kvm_emulate.h>
27#include <asm/kvm_coproc.h>
28#include <asm/kvm_host.h>
29#include <asm/sigcontext.h>
30
31#include "trace.h"
32
33#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
34#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
35
36struct kvm_stats_debugfs_item debugfs_entries[] = {
37 VCPU_STAT(hvc_exit_stat),
38 VCPU_STAT(wfe_exit_stat),
39 VCPU_STAT(wfi_exit_stat),
40 VCPU_STAT(mmio_exit_user),
41 VCPU_STAT(mmio_exit_kernel),
42 VCPU_STAT(exits),
43 { NULL }
44};
45
46int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
47{
48 return 0;
49}
50
51static bool core_reg_offset_is_vreg(u64 off)
52{
53 return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) &&
54 off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr);
55}
56
57static u64 core_reg_offset_from_id(u64 id)
58{
59 return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE);
60}
61
62static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off)
63{
64 int size;
65
66 switch (off) {
67 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
68 KVM_REG_ARM_CORE_REG(regs.regs[30]):
69 case KVM_REG_ARM_CORE_REG(regs.sp):
70 case KVM_REG_ARM_CORE_REG(regs.pc):
71 case KVM_REG_ARM_CORE_REG(regs.pstate):
72 case KVM_REG_ARM_CORE_REG(sp_el1):
73 case KVM_REG_ARM_CORE_REG(elr_el1):
74 case KVM_REG_ARM_CORE_REG(spsr[0]) ...
75 KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]):
76 size = sizeof(__u64);
77 break;
78
79 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
80 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
81 size = sizeof(__uint128_t);
82 break;
83
84 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
85 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
86 size = sizeof(__u32);
87 break;
88
89 default:
90 return -EINVAL;
91 }
92
93 if (!IS_ALIGNED(off, size / sizeof(__u32)))
94 return -EINVAL;
95
96 /*
97 * The KVM_REG_ARM64_SVE regs must be used instead of
98 * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on
99 * SVE-enabled vcpus:
100 */
101 if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off))
102 return -EINVAL;
103
104 return size;
105}
106
107static int validate_core_offset(const struct kvm_vcpu *vcpu,
108 const struct kvm_one_reg *reg)
109{
110 u64 off = core_reg_offset_from_id(reg->id);
111 int size = core_reg_size_from_offset(vcpu, off);
112
113 if (size < 0)
114 return -EINVAL;
115
116 if (KVM_REG_SIZE(reg->id) != size)
117 return -EINVAL;
118
119 return 0;
120}
121
122static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
123{
124 /*
125 * Because the kvm_regs structure is a mix of 32, 64 and
126 * 128bit fields, we index it as if it was a 32bit
127 * array. Hence below, nr_regs is the number of entries, and
128 * off the index in the "array".
129 */
130 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
131 struct kvm_regs *regs = vcpu_gp_regs(vcpu);
132 int nr_regs = sizeof(*regs) / sizeof(__u32);
133 u32 off;
134
135 /* Our ID is an index into the kvm_regs struct. */
136 off = core_reg_offset_from_id(reg->id);
137 if (off >= nr_regs ||
138 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
139 return -ENOENT;
140
141 if (validate_core_offset(vcpu, reg))
142 return -EINVAL;
143
144 if (copy_to_user(uaddr, ((u32 *)regs) + off, KVM_REG_SIZE(reg->id)))
145 return -EFAULT;
146
147 return 0;
148}
149
150static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
151{
152 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
153 struct kvm_regs *regs = vcpu_gp_regs(vcpu);
154 int nr_regs = sizeof(*regs) / sizeof(__u32);
155 __uint128_t tmp;
156 void *valp = &tmp;
157 u64 off;
158 int err = 0;
159
160 /* Our ID is an index into the kvm_regs struct. */
161 off = core_reg_offset_from_id(reg->id);
162 if (off >= nr_regs ||
163 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
164 return -ENOENT;
165
166 if (validate_core_offset(vcpu, reg))
167 return -EINVAL;
168
169 if (KVM_REG_SIZE(reg->id) > sizeof(tmp))
170 return -EINVAL;
171
172 if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) {
173 err = -EFAULT;
174 goto out;
175 }
176
177 if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) {
178 u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK;
179 switch (mode) {
180 case PSR_AA32_MODE_USR:
181 if (!system_supports_32bit_el0())
182 return -EINVAL;
183 break;
184 case PSR_AA32_MODE_FIQ:
185 case PSR_AA32_MODE_IRQ:
186 case PSR_AA32_MODE_SVC:
187 case PSR_AA32_MODE_ABT:
188 case PSR_AA32_MODE_UND:
189 if (!vcpu_el1_is_32bit(vcpu))
190 return -EINVAL;
191 break;
192 case PSR_MODE_EL0t:
193 case PSR_MODE_EL1t:
194 case PSR_MODE_EL1h:
195 if (vcpu_el1_is_32bit(vcpu))
196 return -EINVAL;
197 break;
198 default:
199 err = -EINVAL;
200 goto out;
201 }
202 }
203
204 memcpy((u32 *)regs + off, valp, KVM_REG_SIZE(reg->id));
205out:
206 return err;
207}
208
209#define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64)
210#define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64)
211#define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq)))
212
213static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
214{
215 unsigned int max_vq, vq;
216 u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
217
218 if (!vcpu_has_sve(vcpu))
219 return -ENOENT;
220
221 if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl)))
222 return -EINVAL;
223
224 memset(vqs, 0, sizeof(vqs));
225
226 max_vq = sve_vq_from_vl(vcpu->arch.sve_max_vl);
227 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
228 if (sve_vq_available(vq))
229 vqs[vq_word(vq)] |= vq_mask(vq);
230
231 if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs)))
232 return -EFAULT;
233
234 return 0;
235}
236
237static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
238{
239 unsigned int max_vq, vq;
240 u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
241
242 if (!vcpu_has_sve(vcpu))
243 return -ENOENT;
244
245 if (kvm_arm_vcpu_sve_finalized(vcpu))
246 return -EPERM; /* too late! */
247
248 if (WARN_ON(vcpu->arch.sve_state))
249 return -EINVAL;
250
251 if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs)))
252 return -EFAULT;
253
254 max_vq = 0;
255 for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq)
256 if (vq_present(vqs, vq))
257 max_vq = vq;
258
259 if (max_vq > sve_vq_from_vl(kvm_sve_max_vl))
260 return -EINVAL;
261
262 /*
263 * Vector lengths supported by the host can't currently be
264 * hidden from the guest individually: instead we can only set a
265 * maxmium via ZCR_EL2.LEN. So, make sure the available vector
266 * lengths match the set requested exactly up to the requested
267 * maximum:
268 */
269 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
270 if (vq_present(vqs, vq) != sve_vq_available(vq))
271 return -EINVAL;
272
273 /* Can't run with no vector lengths at all: */
274 if (max_vq < SVE_VQ_MIN)
275 return -EINVAL;
276
277 /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */
278 vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq);
279
280 return 0;
281}
282
283#define SVE_REG_SLICE_SHIFT 0
284#define SVE_REG_SLICE_BITS 5
285#define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS)
286#define SVE_REG_ID_BITS 5
287
288#define SVE_REG_SLICE_MASK \
289 GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1, \
290 SVE_REG_SLICE_SHIFT)
291#define SVE_REG_ID_MASK \
292 GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT)
293
294#define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS)
295
296#define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0))
297#define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0))
298
299/*
300 * Number of register slices required to cover each whole SVE register.
301 * NOTE: Only the first slice every exists, for now.
302 * If you are tempted to modify this, you must also rework sve_reg_to_region()
303 * to match:
304 */
305#define vcpu_sve_slices(vcpu) 1
306
307/* Bounds of a single SVE register slice within vcpu->arch.sve_state */
308struct sve_state_reg_region {
309 unsigned int koffset; /* offset into sve_state in kernel memory */
310 unsigned int klen; /* length in kernel memory */
311 unsigned int upad; /* extra trailing padding in user memory */
312};
313
314/*
315 * Validate SVE register ID and get sanitised bounds for user/kernel SVE
316 * register copy
317 */
318static int sve_reg_to_region(struct sve_state_reg_region *region,
319 struct kvm_vcpu *vcpu,
320 const struct kvm_one_reg *reg)
321{
322 /* reg ID ranges for Z- registers */
323 const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0);
324 const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1,
325 SVE_NUM_SLICES - 1);
326
327 /* reg ID ranges for P- registers and FFR (which are contiguous) */
328 const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0);
329 const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1);
330
331 unsigned int vq;
332 unsigned int reg_num;
333
334 unsigned int reqoffset, reqlen; /* User-requested offset and length */
335 unsigned int maxlen; /* Maxmimum permitted length */
336
337 size_t sve_state_size;
338
339 const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1,
340 SVE_NUM_SLICES - 1);
341
342 /* Verify that the P-regs and FFR really do have contiguous IDs: */
343 BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1);
344
345 /* Verify that we match the UAPI header: */
346 BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES);
347
348 reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
349
350 if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) {
351 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
352 return -ENOENT;
353
354 vq = sve_vq_from_vl(vcpu->arch.sve_max_vl);
355
356 reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) -
357 SVE_SIG_REGS_OFFSET;
358 reqlen = KVM_SVE_ZREG_SIZE;
359 maxlen = SVE_SIG_ZREG_SIZE(vq);
360 } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) {
361 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
362 return -ENOENT;
363
364 vq = sve_vq_from_vl(vcpu->arch.sve_max_vl);
365
366 reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) -
367 SVE_SIG_REGS_OFFSET;
368 reqlen = KVM_SVE_PREG_SIZE;
369 maxlen = SVE_SIG_PREG_SIZE(vq);
370 } else {
371 return -EINVAL;
372 }
373
374 sve_state_size = vcpu_sve_state_size(vcpu);
375 if (WARN_ON(!sve_state_size))
376 return -EINVAL;
377
378 region->koffset = array_index_nospec(reqoffset, sve_state_size);
379 region->klen = min(maxlen, reqlen);
380 region->upad = reqlen - region->klen;
381
382 return 0;
383}
384
385static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
386{
387 int ret;
388 struct sve_state_reg_region region;
389 char __user *uptr = (char __user *)reg->addr;
390
391 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
392 if (reg->id == KVM_REG_ARM64_SVE_VLS)
393 return get_sve_vls(vcpu, reg);
394
395 /* Try to interpret reg ID as an architectural SVE register... */
396 ret = sve_reg_to_region(®ion, vcpu, reg);
397 if (ret)
398 return ret;
399
400 if (!kvm_arm_vcpu_sve_finalized(vcpu))
401 return -EPERM;
402
403 if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset,
404 region.klen) ||
405 clear_user(uptr + region.klen, region.upad))
406 return -EFAULT;
407
408 return 0;
409}
410
411static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
412{
413 int ret;
414 struct sve_state_reg_region region;
415 const char __user *uptr = (const char __user *)reg->addr;
416
417 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
418 if (reg->id == KVM_REG_ARM64_SVE_VLS)
419 return set_sve_vls(vcpu, reg);
420
421 /* Try to interpret reg ID as an architectural SVE register... */
422 ret = sve_reg_to_region(®ion, vcpu, reg);
423 if (ret)
424 return ret;
425
426 if (!kvm_arm_vcpu_sve_finalized(vcpu))
427 return -EPERM;
428
429 if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr,
430 region.klen))
431 return -EFAULT;
432
433 return 0;
434}
435
436int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
437{
438 return -EINVAL;
439}
440
441int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
442{
443 return -EINVAL;
444}
445
446static int copy_core_reg_indices(const struct kvm_vcpu *vcpu,
447 u64 __user *uindices)
448{
449 unsigned int i;
450 int n = 0;
451
452 for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {
453 u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i;
454 int size = core_reg_size_from_offset(vcpu, i);
455
456 if (size < 0)
457 continue;
458
459 switch (size) {
460 case sizeof(__u32):
461 reg |= KVM_REG_SIZE_U32;
462 break;
463
464 case sizeof(__u64):
465 reg |= KVM_REG_SIZE_U64;
466 break;
467
468 case sizeof(__uint128_t):
469 reg |= KVM_REG_SIZE_U128;
470 break;
471
472 default:
473 WARN_ON(1);
474 continue;
475 }
476
477 if (uindices) {
478 if (put_user(reg, uindices))
479 return -EFAULT;
480 uindices++;
481 }
482
483 n++;
484 }
485
486 return n;
487}
488
489static unsigned long num_core_regs(const struct kvm_vcpu *vcpu)
490{
491 return copy_core_reg_indices(vcpu, NULL);
492}
493
494/**
495 * ARM64 versions of the TIMER registers, always available on arm64
496 */
497
498#define NUM_TIMER_REGS 3
499
500static bool is_timer_reg(u64 index)
501{
502 switch (index) {
503 case KVM_REG_ARM_TIMER_CTL:
504 case KVM_REG_ARM_TIMER_CNT:
505 case KVM_REG_ARM_TIMER_CVAL:
506 return true;
507 }
508 return false;
509}
510
511static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
512{
513 if (put_user(KVM_REG_ARM_TIMER_CTL, uindices))
514 return -EFAULT;
515 uindices++;
516 if (put_user(KVM_REG_ARM_TIMER_CNT, uindices))
517 return -EFAULT;
518 uindices++;
519 if (put_user(KVM_REG_ARM_TIMER_CVAL, uindices))
520 return -EFAULT;
521
522 return 0;
523}
524
525static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
526{
527 void __user *uaddr = (void __user *)(long)reg->addr;
528 u64 val;
529 int ret;
530
531 ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id));
532 if (ret != 0)
533 return -EFAULT;
534
535 return kvm_arm_timer_set_reg(vcpu, reg->id, val);
536}
537
538static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
539{
540 void __user *uaddr = (void __user *)(long)reg->addr;
541 u64 val;
542
543 val = kvm_arm_timer_get_reg(vcpu, reg->id);
544 return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0;
545}
546
547static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)
548{
549 const unsigned int slices = vcpu_sve_slices(vcpu);
550
551 if (!vcpu_has_sve(vcpu))
552 return 0;
553
554 /* Policed by KVM_GET_REG_LIST: */
555 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
556
557 return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */)
558 + 1; /* KVM_REG_ARM64_SVE_VLS */
559}
560
561static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu,
562 u64 __user *uindices)
563{
564 const unsigned int slices = vcpu_sve_slices(vcpu);
565 u64 reg;
566 unsigned int i, n;
567 int num_regs = 0;
568
569 if (!vcpu_has_sve(vcpu))
570 return 0;
571
572 /* Policed by KVM_GET_REG_LIST: */
573 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
574
575 /*
576 * Enumerate this first, so that userspace can save/restore in
577 * the order reported by KVM_GET_REG_LIST:
578 */
579 reg = KVM_REG_ARM64_SVE_VLS;
580 if (put_user(reg, uindices++))
581 return -EFAULT;
582 ++num_regs;
583
584 for (i = 0; i < slices; i++) {
585 for (n = 0; n < SVE_NUM_ZREGS; n++) {
586 reg = KVM_REG_ARM64_SVE_ZREG(n, i);
587 if (put_user(reg, uindices++))
588 return -EFAULT;
589 num_regs++;
590 }
591
592 for (n = 0; n < SVE_NUM_PREGS; n++) {
593 reg = KVM_REG_ARM64_SVE_PREG(n, i);
594 if (put_user(reg, uindices++))
595 return -EFAULT;
596 num_regs++;
597 }
598
599 reg = KVM_REG_ARM64_SVE_FFR(i);
600 if (put_user(reg, uindices++))
601 return -EFAULT;
602 num_regs++;
603 }
604
605 return num_regs;
606}
607
608/**
609 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG
610 *
611 * This is for all registers.
612 */
613unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
614{
615 unsigned long res = 0;
616
617 res += num_core_regs(vcpu);
618 res += num_sve_regs(vcpu);
619 res += kvm_arm_num_sys_reg_descs(vcpu);
620 res += kvm_arm_get_fw_num_regs(vcpu);
621 res += NUM_TIMER_REGS;
622
623 return res;
624}
625
626/**
627 * kvm_arm_copy_reg_indices - get indices of all registers.
628 *
629 * We do core registers right here, then we append system regs.
630 */
631int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
632{
633 int ret;
634
635 ret = copy_core_reg_indices(vcpu, uindices);
636 if (ret < 0)
637 return ret;
638 uindices += ret;
639
640 ret = copy_sve_reg_indices(vcpu, uindices);
641 if (ret < 0)
642 return ret;
643 uindices += ret;
644
645 ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices);
646 if (ret < 0)
647 return ret;
648 uindices += kvm_arm_get_fw_num_regs(vcpu);
649
650 ret = copy_timer_indices(vcpu, uindices);
651 if (ret < 0)
652 return ret;
653 uindices += NUM_TIMER_REGS;
654
655 return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
656}
657
658int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
659{
660 /* We currently use nothing arch-specific in upper 32 bits */
661 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
662 return -EINVAL;
663
664 switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
665 case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg);
666 case KVM_REG_ARM_FW: return kvm_arm_get_fw_reg(vcpu, reg);
667 case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg);
668 }
669
670 if (is_timer_reg(reg->id))
671 return get_timer_reg(vcpu, reg);
672
673 return kvm_arm_sys_reg_get_reg(vcpu, reg);
674}
675
676int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
677{
678 /* We currently use nothing arch-specific in upper 32 bits */
679 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
680 return -EINVAL;
681
682 switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
683 case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg);
684 case KVM_REG_ARM_FW: return kvm_arm_set_fw_reg(vcpu, reg);
685 case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg);
686 }
687
688 if (is_timer_reg(reg->id))
689 return set_timer_reg(vcpu, reg);
690
691 return kvm_arm_sys_reg_set_reg(vcpu, reg);
692}
693
694int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
695 struct kvm_sregs *sregs)
696{
697 return -EINVAL;
698}
699
700int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
701 struct kvm_sregs *sregs)
702{
703 return -EINVAL;
704}
705
706int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
707 struct kvm_vcpu_events *events)
708{
709 events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE);
710 events->exception.serror_has_esr = cpus_have_const_cap(ARM64_HAS_RAS_EXTN);
711
712 if (events->exception.serror_pending && events->exception.serror_has_esr)
713 events->exception.serror_esr = vcpu_get_vsesr(vcpu);
714
715 return 0;
716}
717
718int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
719 struct kvm_vcpu_events *events)
720{
721 bool serror_pending = events->exception.serror_pending;
722 bool has_esr = events->exception.serror_has_esr;
723
724 if (serror_pending && has_esr) {
725 if (!cpus_have_const_cap(ARM64_HAS_RAS_EXTN))
726 return -EINVAL;
727
728 if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK))
729 kvm_set_sei_esr(vcpu, events->exception.serror_esr);
730 else
731 return -EINVAL;
732 } else if (serror_pending) {
733 kvm_inject_vabt(vcpu);
734 }
735
736 return 0;
737}
738
739int __attribute_const__ kvm_target_cpu(void)
740{
741 unsigned long implementor = read_cpuid_implementor();
742 unsigned long part_number = read_cpuid_part_number();
743
744 switch (implementor) {
745 case ARM_CPU_IMP_ARM:
746 switch (part_number) {
747 case ARM_CPU_PART_AEM_V8:
748 return KVM_ARM_TARGET_AEM_V8;
749 case ARM_CPU_PART_FOUNDATION:
750 return KVM_ARM_TARGET_FOUNDATION_V8;
751 case ARM_CPU_PART_CORTEX_A53:
752 return KVM_ARM_TARGET_CORTEX_A53;
753 case ARM_CPU_PART_CORTEX_A57:
754 return KVM_ARM_TARGET_CORTEX_A57;
755 }
756 break;
757 case ARM_CPU_IMP_APM:
758 switch (part_number) {
759 case APM_CPU_PART_POTENZA:
760 return KVM_ARM_TARGET_XGENE_POTENZA;
761 }
762 break;
763 }
764
765 /* Return a default generic target */
766 return KVM_ARM_TARGET_GENERIC_V8;
767}
768
769int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init)
770{
771 int target = kvm_target_cpu();
772
773 if (target < 0)
774 return -ENODEV;
775
776 memset(init, 0, sizeof(*init));
777
778 /*
779 * For now, we don't return any features.
780 * In future, we might use features to return target
781 * specific features available for the preferred
782 * target type.
783 */
784 init->target = (__u32)target;
785
786 return 0;
787}
788
789int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
790{
791 return -EINVAL;
792}
793
794int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
795{
796 return -EINVAL;
797}
798
799int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
800 struct kvm_translation *tr)
801{
802 return -EINVAL;
803}
804
805#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \
806 KVM_GUESTDBG_USE_SW_BP | \
807 KVM_GUESTDBG_USE_HW | \
808 KVM_GUESTDBG_SINGLESTEP)
809
810/**
811 * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging
812 * @kvm: pointer to the KVM struct
813 * @kvm_guest_debug: the ioctl data buffer
814 *
815 * This sets up and enables the VM for guest debugging. Userspace
816 * passes in a control flag to enable different debug types and
817 * potentially other architecture specific information in the rest of
818 * the structure.
819 */
820int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
821 struct kvm_guest_debug *dbg)
822{
823 int ret = 0;
824
825 trace_kvm_set_guest_debug(vcpu, dbg->control);
826
827 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) {
828 ret = -EINVAL;
829 goto out;
830 }
831
832 if (dbg->control & KVM_GUESTDBG_ENABLE) {
833 vcpu->guest_debug = dbg->control;
834
835 /* Hardware assisted Break and Watch points */
836 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) {
837 vcpu->arch.external_debug_state = dbg->arch;
838 }
839
840 } else {
841 /* If not enabled clear all flags */
842 vcpu->guest_debug = 0;
843 }
844
845out:
846 return ret;
847}
848
849int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
850 struct kvm_device_attr *attr)
851{
852 int ret;
853
854 switch (attr->group) {
855 case KVM_ARM_VCPU_PMU_V3_CTRL:
856 ret = kvm_arm_pmu_v3_set_attr(vcpu, attr);
857 break;
858 case KVM_ARM_VCPU_TIMER_CTRL:
859 ret = kvm_arm_timer_set_attr(vcpu, attr);
860 break;
861 default:
862 ret = -ENXIO;
863 break;
864 }
865
866 return ret;
867}
868
869int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
870 struct kvm_device_attr *attr)
871{
872 int ret;
873
874 switch (attr->group) {
875 case KVM_ARM_VCPU_PMU_V3_CTRL:
876 ret = kvm_arm_pmu_v3_get_attr(vcpu, attr);
877 break;
878 case KVM_ARM_VCPU_TIMER_CTRL:
879 ret = kvm_arm_timer_get_attr(vcpu, attr);
880 break;
881 default:
882 ret = -ENXIO;
883 break;
884 }
885
886 return ret;
887}
888
889int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
890 struct kvm_device_attr *attr)
891{
892 int ret;
893
894 switch (attr->group) {
895 case KVM_ARM_VCPU_PMU_V3_CTRL:
896 ret = kvm_arm_pmu_v3_has_attr(vcpu, attr);
897 break;
898 case KVM_ARM_VCPU_TIMER_CTRL:
899 ret = kvm_arm_timer_has_attr(vcpu, attr);
900 break;
901 default:
902 ret = -ENXIO;
903 break;
904 }
905
906 return ret;
907}
1/*
2 * Copyright (C) 2012,2013 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * Derived from arch/arm/kvm/guest.c:
6 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
7 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 */
21
22#include <linux/errno.h>
23#include <linux/err.h>
24#include <linux/kvm_host.h>
25#include <linux/module.h>
26#include <linux/vmalloc.h>
27#include <linux/fs.h>
28#include <asm/cputype.h>
29#include <asm/uaccess.h>
30#include <asm/kvm.h>
31#include <asm/kvm_emulate.h>
32#include <asm/kvm_coproc.h>
33
34#include "trace.h"
35
36#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
37#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
38
39struct kvm_stats_debugfs_item debugfs_entries[] = {
40 VCPU_STAT(hvc_exit_stat),
41 VCPU_STAT(wfe_exit_stat),
42 VCPU_STAT(wfi_exit_stat),
43 VCPU_STAT(mmio_exit_user),
44 VCPU_STAT(mmio_exit_kernel),
45 VCPU_STAT(exits),
46 { NULL }
47};
48
49int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
50{
51 return 0;
52}
53
54static u64 core_reg_offset_from_id(u64 id)
55{
56 return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE);
57}
58
59static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
60{
61 /*
62 * Because the kvm_regs structure is a mix of 32, 64 and
63 * 128bit fields, we index it as if it was a 32bit
64 * array. Hence below, nr_regs is the number of entries, and
65 * off the index in the "array".
66 */
67 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
68 struct kvm_regs *regs = vcpu_gp_regs(vcpu);
69 int nr_regs = sizeof(*regs) / sizeof(__u32);
70 u32 off;
71
72 /* Our ID is an index into the kvm_regs struct. */
73 off = core_reg_offset_from_id(reg->id);
74 if (off >= nr_regs ||
75 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
76 return -ENOENT;
77
78 if (copy_to_user(uaddr, ((u32 *)regs) + off, KVM_REG_SIZE(reg->id)))
79 return -EFAULT;
80
81 return 0;
82}
83
84static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
85{
86 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
87 struct kvm_regs *regs = vcpu_gp_regs(vcpu);
88 int nr_regs = sizeof(*regs) / sizeof(__u32);
89 __uint128_t tmp;
90 void *valp = &tmp;
91 u64 off;
92 int err = 0;
93
94 /* Our ID is an index into the kvm_regs struct. */
95 off = core_reg_offset_from_id(reg->id);
96 if (off >= nr_regs ||
97 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
98 return -ENOENT;
99
100 if (KVM_REG_SIZE(reg->id) > sizeof(tmp))
101 return -EINVAL;
102
103 if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) {
104 err = -EFAULT;
105 goto out;
106 }
107
108 if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) {
109 u32 mode = (*(u32 *)valp) & COMPAT_PSR_MODE_MASK;
110 switch (mode) {
111 case COMPAT_PSR_MODE_USR:
112 case COMPAT_PSR_MODE_FIQ:
113 case COMPAT_PSR_MODE_IRQ:
114 case COMPAT_PSR_MODE_SVC:
115 case COMPAT_PSR_MODE_ABT:
116 case COMPAT_PSR_MODE_UND:
117 case PSR_MODE_EL0t:
118 case PSR_MODE_EL1t:
119 case PSR_MODE_EL1h:
120 break;
121 default:
122 err = -EINVAL;
123 goto out;
124 }
125 }
126
127 memcpy((u32 *)regs + off, valp, KVM_REG_SIZE(reg->id));
128out:
129 return err;
130}
131
132int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
133{
134 return -EINVAL;
135}
136
137int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
138{
139 return -EINVAL;
140}
141
142static unsigned long num_core_regs(void)
143{
144 return sizeof(struct kvm_regs) / sizeof(__u32);
145}
146
147/**
148 * ARM64 versions of the TIMER registers, always available on arm64
149 */
150
151#define NUM_TIMER_REGS 3
152
153static bool is_timer_reg(u64 index)
154{
155 switch (index) {
156 case KVM_REG_ARM_TIMER_CTL:
157 case KVM_REG_ARM_TIMER_CNT:
158 case KVM_REG_ARM_TIMER_CVAL:
159 return true;
160 }
161 return false;
162}
163
164static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
165{
166 if (put_user(KVM_REG_ARM_TIMER_CTL, uindices))
167 return -EFAULT;
168 uindices++;
169 if (put_user(KVM_REG_ARM_TIMER_CNT, uindices))
170 return -EFAULT;
171 uindices++;
172 if (put_user(KVM_REG_ARM_TIMER_CVAL, uindices))
173 return -EFAULT;
174
175 return 0;
176}
177
178static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
179{
180 void __user *uaddr = (void __user *)(long)reg->addr;
181 u64 val;
182 int ret;
183
184 ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id));
185 if (ret != 0)
186 return -EFAULT;
187
188 return kvm_arm_timer_set_reg(vcpu, reg->id, val);
189}
190
191static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
192{
193 void __user *uaddr = (void __user *)(long)reg->addr;
194 u64 val;
195
196 val = kvm_arm_timer_get_reg(vcpu, reg->id);
197 return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0;
198}
199
200/**
201 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG
202 *
203 * This is for all registers.
204 */
205unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
206{
207 return num_core_regs() + kvm_arm_num_sys_reg_descs(vcpu)
208 + NUM_TIMER_REGS;
209}
210
211/**
212 * kvm_arm_copy_reg_indices - get indices of all registers.
213 *
214 * We do core registers right here, then we apppend system regs.
215 */
216int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
217{
218 unsigned int i;
219 const u64 core_reg = KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE;
220 int ret;
221
222 for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {
223 if (put_user(core_reg | i, uindices))
224 return -EFAULT;
225 uindices++;
226 }
227
228 ret = copy_timer_indices(vcpu, uindices);
229 if (ret)
230 return ret;
231 uindices += NUM_TIMER_REGS;
232
233 return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
234}
235
236int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
237{
238 /* We currently use nothing arch-specific in upper 32 bits */
239 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
240 return -EINVAL;
241
242 /* Register group 16 means we want a core register. */
243 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_CORE)
244 return get_core_reg(vcpu, reg);
245
246 if (is_timer_reg(reg->id))
247 return get_timer_reg(vcpu, reg);
248
249 return kvm_arm_sys_reg_get_reg(vcpu, reg);
250}
251
252int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
253{
254 /* We currently use nothing arch-specific in upper 32 bits */
255 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
256 return -EINVAL;
257
258 /* Register group 16 means we set a core register. */
259 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_CORE)
260 return set_core_reg(vcpu, reg);
261
262 if (is_timer_reg(reg->id))
263 return set_timer_reg(vcpu, reg);
264
265 return kvm_arm_sys_reg_set_reg(vcpu, reg);
266}
267
268int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
269 struct kvm_sregs *sregs)
270{
271 return -EINVAL;
272}
273
274int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
275 struct kvm_sregs *sregs)
276{
277 return -EINVAL;
278}
279
280int __attribute_const__ kvm_target_cpu(void)
281{
282 unsigned long implementor = read_cpuid_implementor();
283 unsigned long part_number = read_cpuid_part_number();
284
285 switch (implementor) {
286 case ARM_CPU_IMP_ARM:
287 switch (part_number) {
288 case ARM_CPU_PART_AEM_V8:
289 return KVM_ARM_TARGET_AEM_V8;
290 case ARM_CPU_PART_FOUNDATION:
291 return KVM_ARM_TARGET_FOUNDATION_V8;
292 case ARM_CPU_PART_CORTEX_A53:
293 return KVM_ARM_TARGET_CORTEX_A53;
294 case ARM_CPU_PART_CORTEX_A57:
295 return KVM_ARM_TARGET_CORTEX_A57;
296 };
297 break;
298 case ARM_CPU_IMP_APM:
299 switch (part_number) {
300 case APM_CPU_PART_POTENZA:
301 return KVM_ARM_TARGET_XGENE_POTENZA;
302 };
303 break;
304 };
305
306 /* Return a default generic target */
307 return KVM_ARM_TARGET_GENERIC_V8;
308}
309
310int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init)
311{
312 int target = kvm_target_cpu();
313
314 if (target < 0)
315 return -ENODEV;
316
317 memset(init, 0, sizeof(*init));
318
319 /*
320 * For now, we don't return any features.
321 * In future, we might use features to return target
322 * specific features available for the preferred
323 * target type.
324 */
325 init->target = (__u32)target;
326
327 return 0;
328}
329
330int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
331{
332 return -EINVAL;
333}
334
335int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
336{
337 return -EINVAL;
338}
339
340int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
341 struct kvm_translation *tr)
342{
343 return -EINVAL;
344}
345
346#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \
347 KVM_GUESTDBG_USE_SW_BP | \
348 KVM_GUESTDBG_USE_HW | \
349 KVM_GUESTDBG_SINGLESTEP)
350
351/**
352 * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging
353 * @kvm: pointer to the KVM struct
354 * @kvm_guest_debug: the ioctl data buffer
355 *
356 * This sets up and enables the VM for guest debugging. Userspace
357 * passes in a control flag to enable different debug types and
358 * potentially other architecture specific information in the rest of
359 * the structure.
360 */
361int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
362 struct kvm_guest_debug *dbg)
363{
364 trace_kvm_set_guest_debug(vcpu, dbg->control);
365
366 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK)
367 return -EINVAL;
368
369 if (dbg->control & KVM_GUESTDBG_ENABLE) {
370 vcpu->guest_debug = dbg->control;
371
372 /* Hardware assisted Break and Watch points */
373 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) {
374 vcpu->arch.external_debug_state = dbg->arch;
375 }
376
377 } else {
378 /* If not enabled clear all flags */
379 vcpu->guest_debug = 0;
380 }
381 return 0;
382}
383
384int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
385 struct kvm_device_attr *attr)
386{
387 int ret;
388
389 switch (attr->group) {
390 case KVM_ARM_VCPU_PMU_V3_CTRL:
391 ret = kvm_arm_pmu_v3_set_attr(vcpu, attr);
392 break;
393 default:
394 ret = -ENXIO;
395 break;
396 }
397
398 return ret;
399}
400
401int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
402 struct kvm_device_attr *attr)
403{
404 int ret;
405
406 switch (attr->group) {
407 case KVM_ARM_VCPU_PMU_V3_CTRL:
408 ret = kvm_arm_pmu_v3_get_attr(vcpu, attr);
409 break;
410 default:
411 ret = -ENXIO;
412 break;
413 }
414
415 return ret;
416}
417
418int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
419 struct kvm_device_attr *attr)
420{
421 int ret;
422
423 switch (attr->group) {
424 case KVM_ARM_VCPU_PMU_V3_CTRL:
425 ret = kvm_arm_pmu_v3_has_attr(vcpu, attr);
426 break;
427 default:
428 ret = -ENXIO;
429 break;
430 }
431
432 return ret;
433}