Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <linux/cpu.h>
8#include <linux/kvm.h>
9#include <linux/kvm_host.h>
10#include <linux/interrupt.h>
11#include <linux/irq.h>
12#include <linux/uaccess.h>
13
14#include <clocksource/arm_arch_timer.h>
15#include <asm/arch_timer.h>
16#include <asm/kvm_emulate.h>
17#include <asm/kvm_hyp.h>
18
19#include <kvm/arm_vgic.h>
20#include <kvm/arm_arch_timer.h>
21
22#include "trace.h"
23
24static struct timecounter *timecounter;
25static unsigned int host_vtimer_irq;
26static unsigned int host_ptimer_irq;
27static u32 host_vtimer_irq_flags;
28static u32 host_ptimer_irq_flags;
29
30static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);
31
32static const struct kvm_irq_level default_ptimer_irq = {
33 .irq = 30,
34 .level = 1,
35};
36
37static const struct kvm_irq_level default_vtimer_irq = {
38 .irq = 27,
39 .level = 1,
40};
41
42static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
43static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
44 struct arch_timer_context *timer_ctx);
45static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
46static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
47 struct arch_timer_context *timer,
48 enum kvm_arch_timer_regs treg,
49 u64 val);
50static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
51 struct arch_timer_context *timer,
52 enum kvm_arch_timer_regs treg);
53
54u32 timer_get_ctl(struct arch_timer_context *ctxt)
55{
56 struct kvm_vcpu *vcpu = ctxt->vcpu;
57
58 switch(arch_timer_ctx_index(ctxt)) {
59 case TIMER_VTIMER:
60 return __vcpu_sys_reg(vcpu, CNTV_CTL_EL0);
61 case TIMER_PTIMER:
62 return __vcpu_sys_reg(vcpu, CNTP_CTL_EL0);
63 default:
64 WARN_ON(1);
65 return 0;
66 }
67}
68
69u64 timer_get_cval(struct arch_timer_context *ctxt)
70{
71 struct kvm_vcpu *vcpu = ctxt->vcpu;
72
73 switch(arch_timer_ctx_index(ctxt)) {
74 case TIMER_VTIMER:
75 return __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);
76 case TIMER_PTIMER:
77 return __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);
78 default:
79 WARN_ON(1);
80 return 0;
81 }
82}
83
84static u64 timer_get_offset(struct arch_timer_context *ctxt)
85{
86 struct kvm_vcpu *vcpu = ctxt->vcpu;
87
88 switch(arch_timer_ctx_index(ctxt)) {
89 case TIMER_VTIMER:
90 return __vcpu_sys_reg(vcpu, CNTVOFF_EL2);
91 default:
92 return 0;
93 }
94}
95
96static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl)
97{
98 struct kvm_vcpu *vcpu = ctxt->vcpu;
99
100 switch(arch_timer_ctx_index(ctxt)) {
101 case TIMER_VTIMER:
102 __vcpu_sys_reg(vcpu, CNTV_CTL_EL0) = ctl;
103 break;
104 case TIMER_PTIMER:
105 __vcpu_sys_reg(vcpu, CNTP_CTL_EL0) = ctl;
106 break;
107 default:
108 WARN_ON(1);
109 }
110}
111
112static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval)
113{
114 struct kvm_vcpu *vcpu = ctxt->vcpu;
115
116 switch(arch_timer_ctx_index(ctxt)) {
117 case TIMER_VTIMER:
118 __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0) = cval;
119 break;
120 case TIMER_PTIMER:
121 __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = cval;
122 break;
123 default:
124 WARN_ON(1);
125 }
126}
127
128static void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
129{
130 struct kvm_vcpu *vcpu = ctxt->vcpu;
131
132 switch(arch_timer_ctx_index(ctxt)) {
133 case TIMER_VTIMER:
134 __vcpu_sys_reg(vcpu, CNTVOFF_EL2) = offset;
135 break;
136 default:
137 WARN(offset, "timer %ld\n", arch_timer_ctx_index(ctxt));
138 }
139}
140
141u64 kvm_phys_timer_read(void)
142{
143 return timecounter->cc->read(timecounter->cc);
144}
145
146static void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map)
147{
148 if (has_vhe()) {
149 map->direct_vtimer = vcpu_vtimer(vcpu);
150 map->direct_ptimer = vcpu_ptimer(vcpu);
151 map->emul_ptimer = NULL;
152 } else {
153 map->direct_vtimer = vcpu_vtimer(vcpu);
154 map->direct_ptimer = NULL;
155 map->emul_ptimer = vcpu_ptimer(vcpu);
156 }
157
158 trace_kvm_get_timer_map(vcpu->vcpu_id, map);
159}
160
161static inline bool userspace_irqchip(struct kvm *kvm)
162{
163 return static_branch_unlikely(&userspace_irqchip_in_use) &&
164 unlikely(!irqchip_in_kernel(kvm));
165}
166
167static void soft_timer_start(struct hrtimer *hrt, u64 ns)
168{
169 hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
170 HRTIMER_MODE_ABS_HARD);
171}
172
173static void soft_timer_cancel(struct hrtimer *hrt)
174{
175 hrtimer_cancel(hrt);
176}
177
178static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
179{
180 struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
181 struct arch_timer_context *ctx;
182 struct timer_map map;
183
184 /*
185 * We may see a timer interrupt after vcpu_put() has been called which
186 * sets the CPU's vcpu pointer to NULL, because even though the timer
187 * has been disabled in timer_save_state(), the hardware interrupt
188 * signal may not have been retired from the interrupt controller yet.
189 */
190 if (!vcpu)
191 return IRQ_HANDLED;
192
193 get_timer_map(vcpu, &map);
194
195 if (irq == host_vtimer_irq)
196 ctx = map.direct_vtimer;
197 else
198 ctx = map.direct_ptimer;
199
200 if (kvm_timer_should_fire(ctx))
201 kvm_timer_update_irq(vcpu, true, ctx);
202
203 if (userspace_irqchip(vcpu->kvm) &&
204 !static_branch_unlikely(&has_gic_active_state))
205 disable_percpu_irq(host_vtimer_irq);
206
207 return IRQ_HANDLED;
208}
209
210static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
211{
212 u64 cval, now;
213
214 cval = timer_get_cval(timer_ctx);
215 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
216
217 if (now < cval) {
218 u64 ns;
219
220 ns = cyclecounter_cyc2ns(timecounter->cc,
221 cval - now,
222 timecounter->mask,
223 &timecounter->frac);
224 return ns;
225 }
226
227 return 0;
228}
229
230static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
231{
232 WARN_ON(timer_ctx && timer_ctx->loaded);
233 return timer_ctx &&
234 ((timer_get_ctl(timer_ctx) &
235 (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE);
236}
237
238/*
239 * Returns the earliest expiration time in ns among guest timers.
240 * Note that it will return 0 if none of timers can fire.
241 */
242static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
243{
244 u64 min_delta = ULLONG_MAX;
245 int i;
246
247 for (i = 0; i < NR_KVM_TIMERS; i++) {
248 struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
249
250 WARN(ctx->loaded, "timer %d loaded\n", i);
251 if (kvm_timer_irq_can_fire(ctx))
252 min_delta = min(min_delta, kvm_timer_compute_delta(ctx));
253 }
254
255 /* If none of timers can fire, then return 0 */
256 if (min_delta == ULLONG_MAX)
257 return 0;
258
259 return min_delta;
260}
261
262static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
263{
264 struct arch_timer_cpu *timer;
265 struct kvm_vcpu *vcpu;
266 u64 ns;
267
268 timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
269 vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
270
271 /*
272 * Check that the timer has really expired from the guest's
273 * PoV (NTP on the host may have forced it to expire
274 * early). If we should have slept longer, restart it.
275 */
276 ns = kvm_timer_earliest_exp(vcpu);
277 if (unlikely(ns)) {
278 hrtimer_forward_now(hrt, ns_to_ktime(ns));
279 return HRTIMER_RESTART;
280 }
281
282 kvm_vcpu_wake_up(vcpu);
283 return HRTIMER_NORESTART;
284}
285
286static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt)
287{
288 struct arch_timer_context *ctx;
289 struct kvm_vcpu *vcpu;
290 u64 ns;
291
292 ctx = container_of(hrt, struct arch_timer_context, hrtimer);
293 vcpu = ctx->vcpu;
294
295 trace_kvm_timer_hrtimer_expire(ctx);
296
297 /*
298 * Check that the timer has really expired from the guest's
299 * PoV (NTP on the host may have forced it to expire
300 * early). If not ready, schedule for a later time.
301 */
302 ns = kvm_timer_compute_delta(ctx);
303 if (unlikely(ns)) {
304 hrtimer_forward_now(hrt, ns_to_ktime(ns));
305 return HRTIMER_RESTART;
306 }
307
308 kvm_timer_update_irq(vcpu, true, ctx);
309 return HRTIMER_NORESTART;
310}
311
312static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
313{
314 enum kvm_arch_timers index;
315 u64 cval, now;
316
317 if (!timer_ctx)
318 return false;
319
320 index = arch_timer_ctx_index(timer_ctx);
321
322 if (timer_ctx->loaded) {
323 u32 cnt_ctl = 0;
324
325 switch (index) {
326 case TIMER_VTIMER:
327 cnt_ctl = read_sysreg_el0(SYS_CNTV_CTL);
328 break;
329 case TIMER_PTIMER:
330 cnt_ctl = read_sysreg_el0(SYS_CNTP_CTL);
331 break;
332 case NR_KVM_TIMERS:
333 /* GCC is braindead */
334 cnt_ctl = 0;
335 break;
336 }
337
338 return (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
339 (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
340 !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
341 }
342
343 if (!kvm_timer_irq_can_fire(timer_ctx))
344 return false;
345
346 cval = timer_get_cval(timer_ctx);
347 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
348
349 return cval <= now;
350}
351
352bool kvm_timer_is_pending(struct kvm_vcpu *vcpu)
353{
354 struct timer_map map;
355
356 get_timer_map(vcpu, &map);
357
358 return kvm_timer_should_fire(map.direct_vtimer) ||
359 kvm_timer_should_fire(map.direct_ptimer) ||
360 kvm_timer_should_fire(map.emul_ptimer);
361}
362
363/*
364 * Reflect the timer output level into the kvm_run structure
365 */
366void kvm_timer_update_run(struct kvm_vcpu *vcpu)
367{
368 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
369 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
370 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
371
372 /* Populate the device bitmap with the timer states */
373 regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
374 KVM_ARM_DEV_EL1_PTIMER);
375 if (kvm_timer_should_fire(vtimer))
376 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
377 if (kvm_timer_should_fire(ptimer))
378 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
379}
380
381static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
382 struct arch_timer_context *timer_ctx)
383{
384 int ret;
385
386 timer_ctx->irq.level = new_level;
387 trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
388 timer_ctx->irq.level);
389
390 if (!userspace_irqchip(vcpu->kvm)) {
391 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
392 timer_ctx->irq.irq,
393 timer_ctx->irq.level,
394 timer_ctx);
395 WARN_ON(ret);
396 }
397}
398
399/* Only called for a fully emulated timer */
400static void timer_emulate(struct arch_timer_context *ctx)
401{
402 bool should_fire = kvm_timer_should_fire(ctx);
403
404 trace_kvm_timer_emulate(ctx, should_fire);
405
406 if (should_fire != ctx->irq.level) {
407 kvm_timer_update_irq(ctx->vcpu, should_fire, ctx);
408 return;
409 }
410
411 /*
412 * If the timer can fire now, we don't need to have a soft timer
413 * scheduled for the future. If the timer cannot fire at all,
414 * then we also don't need a soft timer.
415 */
416 if (!kvm_timer_irq_can_fire(ctx)) {
417 soft_timer_cancel(&ctx->hrtimer);
418 return;
419 }
420
421 soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));
422}
423
424static void timer_save_state(struct arch_timer_context *ctx)
425{
426 struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
427 enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
428 unsigned long flags;
429
430 if (!timer->enabled)
431 return;
432
433 local_irq_save(flags);
434
435 if (!ctx->loaded)
436 goto out;
437
438 switch (index) {
439 case TIMER_VTIMER:
440 timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL));
441 timer_set_cval(ctx, read_sysreg_el0(SYS_CNTV_CVAL));
442
443 /* Disable the timer */
444 write_sysreg_el0(0, SYS_CNTV_CTL);
445 isb();
446
447 break;
448 case TIMER_PTIMER:
449 timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL));
450 timer_set_cval(ctx, read_sysreg_el0(SYS_CNTP_CVAL));
451
452 /* Disable the timer */
453 write_sysreg_el0(0, SYS_CNTP_CTL);
454 isb();
455
456 break;
457 case NR_KVM_TIMERS:
458 BUG();
459 }
460
461 trace_kvm_timer_save_state(ctx);
462
463 ctx->loaded = false;
464out:
465 local_irq_restore(flags);
466}
467
468/*
469 * Schedule the background timer before calling kvm_vcpu_block, so that this
470 * thread is removed from its waitqueue and made runnable when there's a timer
471 * interrupt to handle.
472 */
473static void kvm_timer_blocking(struct kvm_vcpu *vcpu)
474{
475 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
476 struct timer_map map;
477
478 get_timer_map(vcpu, &map);
479
480 /*
481 * If no timers are capable of raising interrupts (disabled or
482 * masked), then there's no more work for us to do.
483 */
484 if (!kvm_timer_irq_can_fire(map.direct_vtimer) &&
485 !kvm_timer_irq_can_fire(map.direct_ptimer) &&
486 !kvm_timer_irq_can_fire(map.emul_ptimer))
487 return;
488
489 /*
490 * At least one guest time will expire. Schedule a background timer.
491 * Set the earliest expiration time among the guest timers.
492 */
493 soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
494}
495
496static void kvm_timer_unblocking(struct kvm_vcpu *vcpu)
497{
498 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
499
500 soft_timer_cancel(&timer->bg_timer);
501}
502
503static void timer_restore_state(struct arch_timer_context *ctx)
504{
505 struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
506 enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
507 unsigned long flags;
508
509 if (!timer->enabled)
510 return;
511
512 local_irq_save(flags);
513
514 if (ctx->loaded)
515 goto out;
516
517 switch (index) {
518 case TIMER_VTIMER:
519 write_sysreg_el0(timer_get_cval(ctx), SYS_CNTV_CVAL);
520 isb();
521 write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTV_CTL);
522 break;
523 case TIMER_PTIMER:
524 write_sysreg_el0(timer_get_cval(ctx), SYS_CNTP_CVAL);
525 isb();
526 write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL);
527 break;
528 case NR_KVM_TIMERS:
529 BUG();
530 }
531
532 trace_kvm_timer_restore_state(ctx);
533
534 ctx->loaded = true;
535out:
536 local_irq_restore(flags);
537}
538
539static void set_cntvoff(u64 cntvoff)
540{
541 kvm_call_hyp(__kvm_timer_set_cntvoff, cntvoff);
542}
543
544static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active)
545{
546 int r;
547 r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active);
548 WARN_ON(r);
549}
550
551static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
552{
553 struct kvm_vcpu *vcpu = ctx->vcpu;
554 bool phys_active = false;
555
556 /*
557 * Update the timer output so that it is likely to match the
558 * state we're about to restore. If the timer expires between
559 * this point and the register restoration, we'll take the
560 * interrupt anyway.
561 */
562 kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx);
563
564 if (irqchip_in_kernel(vcpu->kvm))
565 phys_active = kvm_vgic_map_is_active(vcpu, ctx->irq.irq);
566
567 phys_active |= ctx->irq.level;
568
569 set_timer_irq_phys_active(ctx, phys_active);
570}
571
572static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
573{
574 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
575
576 /*
577 * Update the timer output so that it is likely to match the
578 * state we're about to restore. If the timer expires between
579 * this point and the register restoration, we'll take the
580 * interrupt anyway.
581 */
582 kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer);
583
584 /*
585 * When using a userspace irqchip with the architected timers and a
586 * host interrupt controller that doesn't support an active state, we
587 * must still prevent continuously exiting from the guest, and
588 * therefore mask the physical interrupt by disabling it on the host
589 * interrupt controller when the virtual level is high, such that the
590 * guest can make forward progress. Once we detect the output level
591 * being de-asserted, we unmask the interrupt again so that we exit
592 * from the guest when the timer fires.
593 */
594 if (vtimer->irq.level)
595 disable_percpu_irq(host_vtimer_irq);
596 else
597 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
598}
599
600void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
601{
602 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
603 struct timer_map map;
604
605 if (unlikely(!timer->enabled))
606 return;
607
608 get_timer_map(vcpu, &map);
609
610 if (static_branch_likely(&has_gic_active_state)) {
611 kvm_timer_vcpu_load_gic(map.direct_vtimer);
612 if (map.direct_ptimer)
613 kvm_timer_vcpu_load_gic(map.direct_ptimer);
614 } else {
615 kvm_timer_vcpu_load_nogic(vcpu);
616 }
617
618 set_cntvoff(timer_get_offset(map.direct_vtimer));
619
620 kvm_timer_unblocking(vcpu);
621
622 timer_restore_state(map.direct_vtimer);
623 if (map.direct_ptimer)
624 timer_restore_state(map.direct_ptimer);
625
626 if (map.emul_ptimer)
627 timer_emulate(map.emul_ptimer);
628}
629
630bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
631{
632 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
633 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
634 struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
635 bool vlevel, plevel;
636
637 if (likely(irqchip_in_kernel(vcpu->kvm)))
638 return false;
639
640 vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
641 plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
642
643 return kvm_timer_should_fire(vtimer) != vlevel ||
644 kvm_timer_should_fire(ptimer) != plevel;
645}
646
647void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
648{
649 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
650 struct timer_map map;
651 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
652
653 if (unlikely(!timer->enabled))
654 return;
655
656 get_timer_map(vcpu, &map);
657
658 timer_save_state(map.direct_vtimer);
659 if (map.direct_ptimer)
660 timer_save_state(map.direct_ptimer);
661
662 /*
663 * Cancel soft timer emulation, because the only case where we
664 * need it after a vcpu_put is in the context of a sleeping VCPU, and
665 * in that case we already factor in the deadline for the physical
666 * timer when scheduling the bg_timer.
667 *
668 * In any case, we re-schedule the hrtimer for the physical timer when
669 * coming back to the VCPU thread in kvm_timer_vcpu_load().
670 */
671 if (map.emul_ptimer)
672 soft_timer_cancel(&map.emul_ptimer->hrtimer);
673
674 if (rcuwait_active(wait))
675 kvm_timer_blocking(vcpu);
676
677 /*
678 * The kernel may decide to run userspace after calling vcpu_put, so
679 * we reset cntvoff to 0 to ensure a consistent read between user
680 * accesses to the virtual counter and kernel access to the physical
681 * counter of non-VHE case. For VHE, the virtual counter uses a fixed
682 * virtual offset of zero, so no need to zero CNTVOFF_EL2 register.
683 */
684 set_cntvoff(0);
685}
686
687/*
688 * With a userspace irqchip we have to check if the guest de-asserted the
689 * timer and if so, unmask the timer irq signal on the host interrupt
690 * controller to ensure that we see future timer signals.
691 */
692static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
693{
694 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
695
696 if (!kvm_timer_should_fire(vtimer)) {
697 kvm_timer_update_irq(vcpu, false, vtimer);
698 if (static_branch_likely(&has_gic_active_state))
699 set_timer_irq_phys_active(vtimer, false);
700 else
701 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
702 }
703}
704
705void kvm_timer_sync_user(struct kvm_vcpu *vcpu)
706{
707 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
708
709 if (unlikely(!timer->enabled))
710 return;
711
712 if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
713 unmask_vtimer_irq_user(vcpu);
714}
715
716int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
717{
718 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
719 struct timer_map map;
720
721 get_timer_map(vcpu, &map);
722
723 /*
724 * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
725 * and to 0 for ARMv7. We provide an implementation that always
726 * resets the timer to be disabled and unmasked and is compliant with
727 * the ARMv7 architecture.
728 */
729 timer_set_ctl(vcpu_vtimer(vcpu), 0);
730 timer_set_ctl(vcpu_ptimer(vcpu), 0);
731
732 if (timer->enabled) {
733 kvm_timer_update_irq(vcpu, false, vcpu_vtimer(vcpu));
734 kvm_timer_update_irq(vcpu, false, vcpu_ptimer(vcpu));
735
736 if (irqchip_in_kernel(vcpu->kvm)) {
737 kvm_vgic_reset_mapped_irq(vcpu, map.direct_vtimer->irq.irq);
738 if (map.direct_ptimer)
739 kvm_vgic_reset_mapped_irq(vcpu, map.direct_ptimer->irq.irq);
740 }
741 }
742
743 if (map.emul_ptimer)
744 soft_timer_cancel(&map.emul_ptimer->hrtimer);
745
746 return 0;
747}
748
749/* Make the updates of cntvoff for all vtimer contexts atomic */
750static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
751{
752 int i;
753 struct kvm *kvm = vcpu->kvm;
754 struct kvm_vcpu *tmp;
755
756 mutex_lock(&kvm->lock);
757 kvm_for_each_vcpu(i, tmp, kvm)
758 timer_set_offset(vcpu_vtimer(tmp), cntvoff);
759
760 /*
761 * When called from the vcpu create path, the CPU being created is not
762 * included in the loop above, so we just set it here as well.
763 */
764 timer_set_offset(vcpu_vtimer(vcpu), cntvoff);
765 mutex_unlock(&kvm->lock);
766}
767
768void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
769{
770 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
771 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
772 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
773
774 vtimer->vcpu = vcpu;
775 ptimer->vcpu = vcpu;
776
777 /* Synchronize cntvoff across all vtimers of a VM. */
778 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
779 timer_set_offset(ptimer, 0);
780
781 hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
782 timer->bg_timer.function = kvm_bg_timer_expire;
783
784 hrtimer_init(&vtimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
785 hrtimer_init(&ptimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
786 vtimer->hrtimer.function = kvm_hrtimer_expire;
787 ptimer->hrtimer.function = kvm_hrtimer_expire;
788
789 vtimer->irq.irq = default_vtimer_irq.irq;
790 ptimer->irq.irq = default_ptimer_irq.irq;
791
792 vtimer->host_timer_irq = host_vtimer_irq;
793 ptimer->host_timer_irq = host_ptimer_irq;
794
795 vtimer->host_timer_irq_flags = host_vtimer_irq_flags;
796 ptimer->host_timer_irq_flags = host_ptimer_irq_flags;
797}
798
799static void kvm_timer_init_interrupt(void *info)
800{
801 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
802 enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags);
803}
804
805int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
806{
807 struct arch_timer_context *timer;
808
809 switch (regid) {
810 case KVM_REG_ARM_TIMER_CTL:
811 timer = vcpu_vtimer(vcpu);
812 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
813 break;
814 case KVM_REG_ARM_TIMER_CNT:
815 timer = vcpu_vtimer(vcpu);
816 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
817 break;
818 case KVM_REG_ARM_TIMER_CVAL:
819 timer = vcpu_vtimer(vcpu);
820 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
821 break;
822 case KVM_REG_ARM_PTIMER_CTL:
823 timer = vcpu_ptimer(vcpu);
824 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
825 break;
826 case KVM_REG_ARM_PTIMER_CVAL:
827 timer = vcpu_ptimer(vcpu);
828 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
829 break;
830
831 default:
832 return -1;
833 }
834
835 return 0;
836}
837
838static u64 read_timer_ctl(struct arch_timer_context *timer)
839{
840 /*
841 * Set ISTATUS bit if it's expired.
842 * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is
843 * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit
844 * regardless of ENABLE bit for our implementation convenience.
845 */
846 u32 ctl = timer_get_ctl(timer);
847
848 if (!kvm_timer_compute_delta(timer))
849 ctl |= ARCH_TIMER_CTRL_IT_STAT;
850
851 return ctl;
852}
853
854u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
855{
856 switch (regid) {
857 case KVM_REG_ARM_TIMER_CTL:
858 return kvm_arm_timer_read(vcpu,
859 vcpu_vtimer(vcpu), TIMER_REG_CTL);
860 case KVM_REG_ARM_TIMER_CNT:
861 return kvm_arm_timer_read(vcpu,
862 vcpu_vtimer(vcpu), TIMER_REG_CNT);
863 case KVM_REG_ARM_TIMER_CVAL:
864 return kvm_arm_timer_read(vcpu,
865 vcpu_vtimer(vcpu), TIMER_REG_CVAL);
866 case KVM_REG_ARM_PTIMER_CTL:
867 return kvm_arm_timer_read(vcpu,
868 vcpu_ptimer(vcpu), TIMER_REG_CTL);
869 case KVM_REG_ARM_PTIMER_CNT:
870 return kvm_arm_timer_read(vcpu,
871 vcpu_ptimer(vcpu), TIMER_REG_CNT);
872 case KVM_REG_ARM_PTIMER_CVAL:
873 return kvm_arm_timer_read(vcpu,
874 vcpu_ptimer(vcpu), TIMER_REG_CVAL);
875 }
876 return (u64)-1;
877}
878
879static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
880 struct arch_timer_context *timer,
881 enum kvm_arch_timer_regs treg)
882{
883 u64 val;
884
885 switch (treg) {
886 case TIMER_REG_TVAL:
887 val = timer_get_cval(timer) - kvm_phys_timer_read() + timer_get_offset(timer);
888 val = lower_32_bits(val);
889 break;
890
891 case TIMER_REG_CTL:
892 val = read_timer_ctl(timer);
893 break;
894
895 case TIMER_REG_CVAL:
896 val = timer_get_cval(timer);
897 break;
898
899 case TIMER_REG_CNT:
900 val = kvm_phys_timer_read() - timer_get_offset(timer);
901 break;
902
903 default:
904 BUG();
905 }
906
907 return val;
908}
909
910u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
911 enum kvm_arch_timers tmr,
912 enum kvm_arch_timer_regs treg)
913{
914 u64 val;
915
916 preempt_disable();
917 kvm_timer_vcpu_put(vcpu);
918
919 val = kvm_arm_timer_read(vcpu, vcpu_get_timer(vcpu, tmr), treg);
920
921 kvm_timer_vcpu_load(vcpu);
922 preempt_enable();
923
924 return val;
925}
926
927static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
928 struct arch_timer_context *timer,
929 enum kvm_arch_timer_regs treg,
930 u64 val)
931{
932 switch (treg) {
933 case TIMER_REG_TVAL:
934 timer_set_cval(timer, kvm_phys_timer_read() - timer_get_offset(timer) + (s32)val);
935 break;
936
937 case TIMER_REG_CTL:
938 timer_set_ctl(timer, val & ~ARCH_TIMER_CTRL_IT_STAT);
939 break;
940
941 case TIMER_REG_CVAL:
942 timer_set_cval(timer, val);
943 break;
944
945 default:
946 BUG();
947 }
948}
949
950void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu,
951 enum kvm_arch_timers tmr,
952 enum kvm_arch_timer_regs treg,
953 u64 val)
954{
955 preempt_disable();
956 kvm_timer_vcpu_put(vcpu);
957
958 kvm_arm_timer_write(vcpu, vcpu_get_timer(vcpu, tmr), treg, val);
959
960 kvm_timer_vcpu_load(vcpu);
961 preempt_enable();
962}
963
964static int kvm_timer_starting_cpu(unsigned int cpu)
965{
966 kvm_timer_init_interrupt(NULL);
967 return 0;
968}
969
970static int kvm_timer_dying_cpu(unsigned int cpu)
971{
972 disable_percpu_irq(host_vtimer_irq);
973 return 0;
974}
975
976int kvm_timer_hyp_init(bool has_gic)
977{
978 struct arch_timer_kvm_info *info;
979 int err;
980
981 info = arch_timer_get_kvm_info();
982 timecounter = &info->timecounter;
983
984 if (!timecounter->cc) {
985 kvm_err("kvm_arch_timer: uninitialized timecounter\n");
986 return -ENODEV;
987 }
988
989 /* First, do the virtual EL1 timer irq */
990
991 if (info->virtual_irq <= 0) {
992 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
993 info->virtual_irq);
994 return -ENODEV;
995 }
996 host_vtimer_irq = info->virtual_irq;
997
998 host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
999 if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
1000 host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
1001 kvm_err("Invalid trigger for vtimer IRQ%d, assuming level low\n",
1002 host_vtimer_irq);
1003 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
1004 }
1005
1006 err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
1007 "kvm guest vtimer", kvm_get_running_vcpus());
1008 if (err) {
1009 kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n",
1010 host_vtimer_irq, err);
1011 return err;
1012 }
1013
1014 if (has_gic) {
1015 err = irq_set_vcpu_affinity(host_vtimer_irq,
1016 kvm_get_running_vcpus());
1017 if (err) {
1018 kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
1019 goto out_free_irq;
1020 }
1021
1022 static_branch_enable(&has_gic_active_state);
1023 }
1024
1025 kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
1026
1027 /* Now let's do the physical EL1 timer irq */
1028
1029 if (info->physical_irq > 0) {
1030 host_ptimer_irq = info->physical_irq;
1031 host_ptimer_irq_flags = irq_get_trigger_type(host_ptimer_irq);
1032 if (host_ptimer_irq_flags != IRQF_TRIGGER_HIGH &&
1033 host_ptimer_irq_flags != IRQF_TRIGGER_LOW) {
1034 kvm_err("Invalid trigger for ptimer IRQ%d, assuming level low\n",
1035 host_ptimer_irq);
1036 host_ptimer_irq_flags = IRQF_TRIGGER_LOW;
1037 }
1038
1039 err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler,
1040 "kvm guest ptimer", kvm_get_running_vcpus());
1041 if (err) {
1042 kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n",
1043 host_ptimer_irq, err);
1044 return err;
1045 }
1046
1047 if (has_gic) {
1048 err = irq_set_vcpu_affinity(host_ptimer_irq,
1049 kvm_get_running_vcpus());
1050 if (err) {
1051 kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
1052 goto out_free_irq;
1053 }
1054 }
1055
1056 kvm_debug("physical timer IRQ%d\n", host_ptimer_irq);
1057 } else if (has_vhe()) {
1058 kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n",
1059 info->physical_irq);
1060 err = -ENODEV;
1061 goto out_free_irq;
1062 }
1063
1064 cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
1065 "kvm/arm/timer:starting", kvm_timer_starting_cpu,
1066 kvm_timer_dying_cpu);
1067 return 0;
1068out_free_irq:
1069 free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
1070 return err;
1071}
1072
1073void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
1074{
1075 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1076
1077 soft_timer_cancel(&timer->bg_timer);
1078}
1079
1080static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
1081{
1082 int vtimer_irq, ptimer_irq;
1083 int i, ret;
1084
1085 vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
1086 ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
1087 if (ret)
1088 return false;
1089
1090 ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
1091 ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
1092 if (ret)
1093 return false;
1094
1095 kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
1096 if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
1097 vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
1098 return false;
1099 }
1100
1101 return true;
1102}
1103
1104bool kvm_arch_timer_get_input_level(int vintid)
1105{
1106 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
1107 struct arch_timer_context *timer;
1108
1109 if (vintid == vcpu_vtimer(vcpu)->irq.irq)
1110 timer = vcpu_vtimer(vcpu);
1111 else if (vintid == vcpu_ptimer(vcpu)->irq.irq)
1112 timer = vcpu_ptimer(vcpu);
1113 else
1114 BUG();
1115
1116 return kvm_timer_should_fire(timer);
1117}
1118
1119int kvm_timer_enable(struct kvm_vcpu *vcpu)
1120{
1121 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1122 struct timer_map map;
1123 int ret;
1124
1125 if (timer->enabled)
1126 return 0;
1127
1128 /* Without a VGIC we do not map virtual IRQs to physical IRQs */
1129 if (!irqchip_in_kernel(vcpu->kvm))
1130 goto no_vgic;
1131
1132 if (!vgic_initialized(vcpu->kvm))
1133 return -ENODEV;
1134
1135 if (!timer_irqs_are_valid(vcpu)) {
1136 kvm_debug("incorrectly configured timer irqs\n");
1137 return -EINVAL;
1138 }
1139
1140 get_timer_map(vcpu, &map);
1141
1142 ret = kvm_vgic_map_phys_irq(vcpu,
1143 map.direct_vtimer->host_timer_irq,
1144 map.direct_vtimer->irq.irq,
1145 kvm_arch_timer_get_input_level);
1146 if (ret)
1147 return ret;
1148
1149 if (map.direct_ptimer) {
1150 ret = kvm_vgic_map_phys_irq(vcpu,
1151 map.direct_ptimer->host_timer_irq,
1152 map.direct_ptimer->irq.irq,
1153 kvm_arch_timer_get_input_level);
1154 }
1155
1156 if (ret)
1157 return ret;
1158
1159no_vgic:
1160 timer->enabled = 1;
1161 return 0;
1162}
1163
1164/*
1165 * On VHE system, we only need to configure the EL2 timer trap register once,
1166 * not for every world switch.
1167 * The host kernel runs at EL2 with HCR_EL2.TGE == 1,
1168 * and this makes those bits have no effect for the host kernel execution.
1169 */
1170void kvm_timer_init_vhe(void)
1171{
1172 /* When HCR_EL2.E2H ==1, EL1PCEN and EL1PCTEN are shifted by 10 */
1173 u32 cnthctl_shift = 10;
1174 u64 val;
1175
1176 /*
1177 * VHE systems allow the guest direct access to the EL1 physical
1178 * timer/counter.
1179 */
1180 val = read_sysreg(cnthctl_el2);
1181 val |= (CNTHCTL_EL1PCEN << cnthctl_shift);
1182 val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
1183 write_sysreg(val, cnthctl_el2);
1184}
1185
1186static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
1187{
1188 struct kvm_vcpu *vcpu;
1189 int i;
1190
1191 kvm_for_each_vcpu(i, vcpu, kvm) {
1192 vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
1193 vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
1194 }
1195}
1196
1197int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1198{
1199 int __user *uaddr = (int __user *)(long)attr->addr;
1200 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
1201 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
1202 int irq;
1203
1204 if (!irqchip_in_kernel(vcpu->kvm))
1205 return -EINVAL;
1206
1207 if (get_user(irq, uaddr))
1208 return -EFAULT;
1209
1210 if (!(irq_is_ppi(irq)))
1211 return -EINVAL;
1212
1213 if (vcpu->arch.timer_cpu.enabled)
1214 return -EBUSY;
1215
1216 switch (attr->attr) {
1217 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1218 set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
1219 break;
1220 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1221 set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
1222 break;
1223 default:
1224 return -ENXIO;
1225 }
1226
1227 return 0;
1228}
1229
1230int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1231{
1232 int __user *uaddr = (int __user *)(long)attr->addr;
1233 struct arch_timer_context *timer;
1234 int irq;
1235
1236 switch (attr->attr) {
1237 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1238 timer = vcpu_vtimer(vcpu);
1239 break;
1240 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1241 timer = vcpu_ptimer(vcpu);
1242 break;
1243 default:
1244 return -ENXIO;
1245 }
1246
1247 irq = timer->irq.irq;
1248 return put_user(irq, uaddr);
1249}
1250
1251int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1252{
1253 switch (attr->attr) {
1254 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1255 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1256 return 0;
1257 }
1258
1259 return -ENXIO;
1260}