Loading...
1/*
2 * Xen time implementation.
3 *
4 * This is implemented in terms of a clocksource driver which uses
5 * the hypervisor clock as a nanosecond timebase, and a clockevent
6 * driver which uses the hypervisor's timer mechanism.
7 *
8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
9 */
10#include <linux/kernel.h>
11#include <linux/interrupt.h>
12#include <linux/clocksource.h>
13#include <linux/clockchips.h>
14#include <linux/kernel_stat.h>
15#include <linux/math64.h>
16#include <linux/gfp.h>
17#include <linux/slab.h>
18#include <linux/pvclock_gtod.h>
19
20#include <asm/pvclock.h>
21#include <asm/xen/hypervisor.h>
22#include <asm/xen/hypercall.h>
23
24#include <xen/events.h>
25#include <xen/features.h>
26#include <xen/interface/xen.h>
27#include <xen/interface/vcpu.h>
28
29#include "xen-ops.h"
30
31/* Xen may fire a timer up to this many ns early */
32#define TIMER_SLOP 100000
33#define NS_PER_TICK (1000000000LL / HZ)
34
35/* runstate info updated by Xen */
36static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
37
38/* snapshots of runstate info */
39static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot);
40
41/* unused ns of stolen time */
42static DEFINE_PER_CPU(u64, xen_residual_stolen);
43
44/* return an consistent snapshot of 64-bit time/counter value */
45static u64 get64(const u64 *p)
46{
47 u64 ret;
48
49 if (BITS_PER_LONG < 64) {
50 u32 *p32 = (u32 *)p;
51 u32 h, l;
52
53 /*
54 * Read high then low, and then make sure high is
55 * still the same; this will only loop if low wraps
56 * and carries into high.
57 * XXX some clean way to make this endian-proof?
58 */
59 do {
60 h = p32[1];
61 barrier();
62 l = p32[0];
63 barrier();
64 } while (p32[1] != h);
65
66 ret = (((u64)h) << 32) | l;
67 } else
68 ret = *p;
69
70 return ret;
71}
72
73/*
74 * Runstate accounting
75 */
76static void get_runstate_snapshot(struct vcpu_runstate_info *res)
77{
78 u64 state_time;
79 struct vcpu_runstate_info *state;
80
81 BUG_ON(preemptible());
82
83 state = &__get_cpu_var(xen_runstate);
84
85 /*
86 * The runstate info is always updated by the hypervisor on
87 * the current CPU, so there's no need to use anything
88 * stronger than a compiler barrier when fetching it.
89 */
90 do {
91 state_time = get64(&state->state_entry_time);
92 barrier();
93 *res = *state;
94 barrier();
95 } while (get64(&state->state_entry_time) != state_time);
96}
97
98/* return true when a vcpu could run but has no real cpu to run on */
99bool xen_vcpu_stolen(int vcpu)
100{
101 return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
102}
103
104void xen_setup_runstate_info(int cpu)
105{
106 struct vcpu_register_runstate_memory_area area;
107
108 area.addr.v = &per_cpu(xen_runstate, cpu);
109
110 if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
111 cpu, &area))
112 BUG();
113}
114
115static void do_stolen_accounting(void)
116{
117 struct vcpu_runstate_info state;
118 struct vcpu_runstate_info *snap;
119 s64 runnable, offline, stolen;
120 cputime_t ticks;
121
122 get_runstate_snapshot(&state);
123
124 WARN_ON(state.state != RUNSTATE_running);
125
126 snap = &__get_cpu_var(xen_runstate_snapshot);
127
128 /* work out how much time the VCPU has not been runn*ing* */
129 runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
130 offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
131
132 *snap = state;
133
134 /* Add the appropriate number of ticks of stolen time,
135 including any left-overs from last time. */
136 stolen = runnable + offline + __this_cpu_read(xen_residual_stolen);
137
138 if (stolen < 0)
139 stolen = 0;
140
141 ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
142 __this_cpu_write(xen_residual_stolen, stolen);
143 account_steal_ticks(ticks);
144}
145
146/* Get the TSC speed from Xen */
147static unsigned long xen_tsc_khz(void)
148{
149 struct pvclock_vcpu_time_info *info =
150 &HYPERVISOR_shared_info->vcpu_info[0].time;
151
152 return pvclock_tsc_khz(info);
153}
154
155cycle_t xen_clocksource_read(void)
156{
157 struct pvclock_vcpu_time_info *src;
158 cycle_t ret;
159
160 preempt_disable_notrace();
161 src = &__get_cpu_var(xen_vcpu)->time;
162 ret = pvclock_clocksource_read(src);
163 preempt_enable_notrace();
164 return ret;
165}
166
167static cycle_t xen_clocksource_get_cycles(struct clocksource *cs)
168{
169 return xen_clocksource_read();
170}
171
172static void xen_read_wallclock(struct timespec *ts)
173{
174 struct shared_info *s = HYPERVISOR_shared_info;
175 struct pvclock_wall_clock *wall_clock = &(s->wc);
176 struct pvclock_vcpu_time_info *vcpu_time;
177
178 vcpu_time = &get_cpu_var(xen_vcpu)->time;
179 pvclock_read_wallclock(wall_clock, vcpu_time, ts);
180 put_cpu_var(xen_vcpu);
181}
182
183static void xen_get_wallclock(struct timespec *now)
184{
185 xen_read_wallclock(now);
186}
187
188static int xen_set_wallclock(const struct timespec *now)
189{
190 return -1;
191}
192
193static int xen_pvclock_gtod_notify(struct notifier_block *nb,
194 unsigned long was_set, void *priv)
195{
196 /* Protected by the calling core code serialization */
197 static struct timespec next_sync;
198
199 struct xen_platform_op op;
200 struct timespec now;
201
202 now = __current_kernel_time();
203
204 /*
205 * We only take the expensive HV call when the clock was set
206 * or when the 11 minutes RTC synchronization time elapsed.
207 */
208 if (!was_set && timespec_compare(&now, &next_sync) < 0)
209 return NOTIFY_OK;
210
211 op.cmd = XENPF_settime;
212 op.u.settime.secs = now.tv_sec;
213 op.u.settime.nsecs = now.tv_nsec;
214 op.u.settime.system_time = xen_clocksource_read();
215
216 (void)HYPERVISOR_dom0_op(&op);
217
218 /*
219 * Move the next drift compensation time 11 minutes
220 * ahead. That's emulating the sync_cmos_clock() update for
221 * the hardware RTC.
222 */
223 next_sync = now;
224 next_sync.tv_sec += 11 * 60;
225
226 return NOTIFY_OK;
227}
228
229static struct notifier_block xen_pvclock_gtod_notifier = {
230 .notifier_call = xen_pvclock_gtod_notify,
231};
232
233static struct clocksource xen_clocksource __read_mostly = {
234 .name = "xen",
235 .rating = 400,
236 .read = xen_clocksource_get_cycles,
237 .mask = ~0,
238 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
239};
240
241/*
242 Xen clockevent implementation
243
244 Xen has two clockevent implementations:
245
246 The old timer_op one works with all released versions of Xen prior
247 to version 3.0.4. This version of the hypervisor provides a
248 single-shot timer with nanosecond resolution. However, sharing the
249 same event channel is a 100Hz tick which is delivered while the
250 vcpu is running. We don't care about or use this tick, but it will
251 cause the core time code to think the timer fired too soon, and
252 will end up resetting it each time. It could be filtered, but
253 doing so has complications when the ktime clocksource is not yet
254 the xen clocksource (ie, at boot time).
255
256 The new vcpu_op-based timer interface allows the tick timer period
257 to be changed or turned off. The tick timer is not useful as a
258 periodic timer because events are only delivered to running vcpus.
259 The one-shot timer can report when a timeout is in the past, so
260 set_next_event is capable of returning -ETIME when appropriate.
261 This interface is used when available.
262*/
263
264
265/*
266 Get a hypervisor absolute time. In theory we could maintain an
267 offset between the kernel's time and the hypervisor's time, and
268 apply that to a kernel's absolute timeout. Unfortunately the
269 hypervisor and kernel times can drift even if the kernel is using
270 the Xen clocksource, because ntp can warp the kernel's clocksource.
271*/
272static s64 get_abs_timeout(unsigned long delta)
273{
274 return xen_clocksource_read() + delta;
275}
276
277static void xen_timerop_set_mode(enum clock_event_mode mode,
278 struct clock_event_device *evt)
279{
280 switch (mode) {
281 case CLOCK_EVT_MODE_PERIODIC:
282 /* unsupported */
283 WARN_ON(1);
284 break;
285
286 case CLOCK_EVT_MODE_ONESHOT:
287 case CLOCK_EVT_MODE_RESUME:
288 break;
289
290 case CLOCK_EVT_MODE_UNUSED:
291 case CLOCK_EVT_MODE_SHUTDOWN:
292 HYPERVISOR_set_timer_op(0); /* cancel timeout */
293 break;
294 }
295}
296
297static int xen_timerop_set_next_event(unsigned long delta,
298 struct clock_event_device *evt)
299{
300 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
301
302 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
303 BUG();
304
305 /* We may have missed the deadline, but there's no real way of
306 knowing for sure. If the event was in the past, then we'll
307 get an immediate interrupt. */
308
309 return 0;
310}
311
312static const struct clock_event_device xen_timerop_clockevent = {
313 .name = "xen",
314 .features = CLOCK_EVT_FEAT_ONESHOT,
315
316 .max_delta_ns = 0xffffffff,
317 .min_delta_ns = TIMER_SLOP,
318
319 .mult = 1,
320 .shift = 0,
321 .rating = 500,
322
323 .set_mode = xen_timerop_set_mode,
324 .set_next_event = xen_timerop_set_next_event,
325};
326
327
328
329static void xen_vcpuop_set_mode(enum clock_event_mode mode,
330 struct clock_event_device *evt)
331{
332 int cpu = smp_processor_id();
333
334 switch (mode) {
335 case CLOCK_EVT_MODE_PERIODIC:
336 WARN_ON(1); /* unsupported */
337 break;
338
339 case CLOCK_EVT_MODE_ONESHOT:
340 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
341 BUG();
342 break;
343
344 case CLOCK_EVT_MODE_UNUSED:
345 case CLOCK_EVT_MODE_SHUTDOWN:
346 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
347 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
348 BUG();
349 break;
350 case CLOCK_EVT_MODE_RESUME:
351 break;
352 }
353}
354
355static int xen_vcpuop_set_next_event(unsigned long delta,
356 struct clock_event_device *evt)
357{
358 int cpu = smp_processor_id();
359 struct vcpu_set_singleshot_timer single;
360 int ret;
361
362 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
363
364 single.timeout_abs_ns = get_abs_timeout(delta);
365 single.flags = VCPU_SSHOTTMR_future;
366
367 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
368
369 BUG_ON(ret != 0 && ret != -ETIME);
370
371 return ret;
372}
373
374static const struct clock_event_device xen_vcpuop_clockevent = {
375 .name = "xen",
376 .features = CLOCK_EVT_FEAT_ONESHOT,
377
378 .max_delta_ns = 0xffffffff,
379 .min_delta_ns = TIMER_SLOP,
380
381 .mult = 1,
382 .shift = 0,
383 .rating = 500,
384
385 .set_mode = xen_vcpuop_set_mode,
386 .set_next_event = xen_vcpuop_set_next_event,
387};
388
389static const struct clock_event_device *xen_clockevent =
390 &xen_timerop_clockevent;
391
392struct xen_clock_event_device {
393 struct clock_event_device evt;
394 char *name;
395};
396static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
397
398static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
399{
400 struct clock_event_device *evt = &__get_cpu_var(xen_clock_events).evt;
401 irqreturn_t ret;
402
403 ret = IRQ_NONE;
404 if (evt->event_handler) {
405 evt->event_handler(evt);
406 ret = IRQ_HANDLED;
407 }
408
409 do_stolen_accounting();
410
411 return ret;
412}
413
414void xen_teardown_timer(int cpu)
415{
416 struct clock_event_device *evt;
417 BUG_ON(cpu == 0);
418 evt = &per_cpu(xen_clock_events, cpu).evt;
419
420 if (evt->irq >= 0) {
421 unbind_from_irqhandler(evt->irq, NULL);
422 evt->irq = -1;
423 kfree(per_cpu(xen_clock_events, cpu).name);
424 per_cpu(xen_clock_events, cpu).name = NULL;
425 }
426}
427
428void xen_setup_timer(int cpu)
429{
430 char *name;
431 struct clock_event_device *evt;
432 int irq;
433
434 evt = &per_cpu(xen_clock_events, cpu).evt;
435 WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
436 if (evt->irq >= 0)
437 xen_teardown_timer(cpu);
438
439 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
440
441 name = kasprintf(GFP_KERNEL, "timer%d", cpu);
442 if (!name)
443 name = "<timer kasprintf failed>";
444
445 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
446 IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER|
447 IRQF_FORCE_RESUME,
448 name, NULL);
449 (void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);
450
451 memcpy(evt, xen_clockevent, sizeof(*evt));
452
453 evt->cpumask = cpumask_of(cpu);
454 evt->irq = irq;
455 per_cpu(xen_clock_events, cpu).name = name;
456}
457
458
459void xen_setup_cpu_clockevents(void)
460{
461 BUG_ON(preemptible());
462
463 clockevents_register_device(&__get_cpu_var(xen_clock_events).evt);
464}
465
466void xen_timer_resume(void)
467{
468 int cpu;
469
470 pvclock_resume();
471
472 if (xen_clockevent != &xen_vcpuop_clockevent)
473 return;
474
475 for_each_online_cpu(cpu) {
476 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
477 BUG();
478 }
479}
480
481static const struct pv_time_ops xen_time_ops __initconst = {
482 .sched_clock = xen_clocksource_read,
483};
484
485static void __init xen_time_init(void)
486{
487 int cpu = smp_processor_id();
488 struct timespec tp;
489
490 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
491
492 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
493 /* Successfully turned off 100Hz tick, so we have the
494 vcpuop-based timer interface */
495 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
496 xen_clockevent = &xen_vcpuop_clockevent;
497 }
498
499 /* Set initial system time with full resolution */
500 xen_read_wallclock(&tp);
501 do_settimeofday(&tp);
502
503 setup_force_cpu_cap(X86_FEATURE_TSC);
504
505 xen_setup_runstate_info(cpu);
506 xen_setup_timer(cpu);
507 xen_setup_cpu_clockevents();
508
509 if (xen_initial_domain())
510 pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
511}
512
513void __init xen_init_time_ops(void)
514{
515 pv_time_ops = xen_time_ops;
516
517 x86_init.timers.timer_init = xen_time_init;
518 x86_init.timers.setup_percpu_clockev = x86_init_noop;
519 x86_cpuinit.setup_percpu_clockev = x86_init_noop;
520
521 x86_platform.calibrate_tsc = xen_tsc_khz;
522 x86_platform.get_wallclock = xen_get_wallclock;
523 /* Dom0 uses the native method to set the hardware RTC. */
524 if (!xen_initial_domain())
525 x86_platform.set_wallclock = xen_set_wallclock;
526}
527
528#ifdef CONFIG_XEN_PVHVM
529static void xen_hvm_setup_cpu_clockevents(void)
530{
531 int cpu = smp_processor_id();
532 xen_setup_runstate_info(cpu);
533 /*
534 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
535 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
536 * early bootup and also during CPU hotplug events).
537 */
538 xen_setup_cpu_clockevents();
539}
540
541void __init xen_hvm_init_time_ops(void)
542{
543 /* vector callback is needed otherwise we cannot receive interrupts
544 * on cpu > 0 and at this point we don't know how many cpus are
545 * available */
546 if (!xen_have_vector_callback)
547 return;
548 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
549 printk(KERN_INFO "Xen doesn't support pvclock on HVM,"
550 "disable pv timer\n");
551 return;
552 }
553
554 pv_time_ops = xen_time_ops;
555 x86_init.timers.setup_percpu_clockev = xen_time_init;
556 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
557
558 x86_platform.calibrate_tsc = xen_tsc_khz;
559 x86_platform.get_wallclock = xen_get_wallclock;
560 x86_platform.set_wallclock = xen_set_wallclock;
561}
562#endif
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Xen time implementation.
4 *
5 * This is implemented in terms of a clocksource driver which uses
6 * the hypervisor clock as a nanosecond timebase, and a clockevent
7 * driver which uses the hypervisor's timer mechanism.
8 *
9 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
10 */
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/clocksource.h>
14#include <linux/clockchips.h>
15#include <linux/gfp.h>
16#include <linux/slab.h>
17#include <linux/pvclock_gtod.h>
18#include <linux/timekeeper_internal.h>
19
20#include <asm/pvclock.h>
21#include <asm/xen/hypervisor.h>
22#include <asm/xen/hypercall.h>
23
24#include <xen/events.h>
25#include <xen/features.h>
26#include <xen/interface/xen.h>
27#include <xen/interface/vcpu.h>
28
29#include "xen-ops.h"
30
31/* Minimum amount of time until next clock event fires */
32#define TIMER_SLOP 100000
33
34static u64 xen_sched_clock_offset __read_mostly;
35
36/* Get the TSC speed from Xen */
37static unsigned long xen_tsc_khz(void)
38{
39 struct pvclock_vcpu_time_info *info =
40 &HYPERVISOR_shared_info->vcpu_info[0].time;
41
42 setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
43 return pvclock_tsc_khz(info);
44}
45
46static u64 xen_clocksource_read(void)
47{
48 struct pvclock_vcpu_time_info *src;
49 u64 ret;
50
51 preempt_disable_notrace();
52 src = &__this_cpu_read(xen_vcpu)->time;
53 ret = pvclock_clocksource_read(src);
54 preempt_enable_notrace();
55 return ret;
56}
57
58static u64 xen_clocksource_get_cycles(struct clocksource *cs)
59{
60 return xen_clocksource_read();
61}
62
63static u64 xen_sched_clock(void)
64{
65 return xen_clocksource_read() - xen_sched_clock_offset;
66}
67
68static void xen_read_wallclock(struct timespec64 *ts)
69{
70 struct shared_info *s = HYPERVISOR_shared_info;
71 struct pvclock_wall_clock *wall_clock = &(s->wc);
72 struct pvclock_vcpu_time_info *vcpu_time;
73
74 vcpu_time = &get_cpu_var(xen_vcpu)->time;
75 pvclock_read_wallclock(wall_clock, vcpu_time, ts);
76 put_cpu_var(xen_vcpu);
77}
78
79static void xen_get_wallclock(struct timespec64 *now)
80{
81 xen_read_wallclock(now);
82}
83
84static int xen_set_wallclock(const struct timespec64 *now)
85{
86 return -ENODEV;
87}
88
89static int xen_pvclock_gtod_notify(struct notifier_block *nb,
90 unsigned long was_set, void *priv)
91{
92 /* Protected by the calling core code serialization */
93 static struct timespec64 next_sync;
94
95 struct xen_platform_op op;
96 struct timespec64 now;
97 struct timekeeper *tk = priv;
98 static bool settime64_supported = true;
99 int ret;
100
101 now.tv_sec = tk->xtime_sec;
102 now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
103
104 /*
105 * We only take the expensive HV call when the clock was set
106 * or when the 11 minutes RTC synchronization time elapsed.
107 */
108 if (!was_set && timespec64_compare(&now, &next_sync) < 0)
109 return NOTIFY_OK;
110
111again:
112 if (settime64_supported) {
113 op.cmd = XENPF_settime64;
114 op.u.settime64.mbz = 0;
115 op.u.settime64.secs = now.tv_sec;
116 op.u.settime64.nsecs = now.tv_nsec;
117 op.u.settime64.system_time = xen_clocksource_read();
118 } else {
119 op.cmd = XENPF_settime32;
120 op.u.settime32.secs = now.tv_sec;
121 op.u.settime32.nsecs = now.tv_nsec;
122 op.u.settime32.system_time = xen_clocksource_read();
123 }
124
125 ret = HYPERVISOR_platform_op(&op);
126
127 if (ret == -ENOSYS && settime64_supported) {
128 settime64_supported = false;
129 goto again;
130 }
131 if (ret < 0)
132 return NOTIFY_BAD;
133
134 /*
135 * Move the next drift compensation time 11 minutes
136 * ahead. That's emulating the sync_cmos_clock() update for
137 * the hardware RTC.
138 */
139 next_sync = now;
140 next_sync.tv_sec += 11 * 60;
141
142 return NOTIFY_OK;
143}
144
145static struct notifier_block xen_pvclock_gtod_notifier = {
146 .notifier_call = xen_pvclock_gtod_notify,
147};
148
149static int xen_cs_enable(struct clocksource *cs)
150{
151 vclocks_set_used(VDSO_CLOCKMODE_PVCLOCK);
152 return 0;
153}
154
155static struct clocksource xen_clocksource __read_mostly = {
156 .name = "xen",
157 .rating = 400,
158 .read = xen_clocksource_get_cycles,
159 .mask = CLOCKSOURCE_MASK(64),
160 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
161 .enable = xen_cs_enable,
162};
163
164/*
165 Xen clockevent implementation
166
167 Xen has two clockevent implementations:
168
169 The old timer_op one works with all released versions of Xen prior
170 to version 3.0.4. This version of the hypervisor provides a
171 single-shot timer with nanosecond resolution. However, sharing the
172 same event channel is a 100Hz tick which is delivered while the
173 vcpu is running. We don't care about or use this tick, but it will
174 cause the core time code to think the timer fired too soon, and
175 will end up resetting it each time. It could be filtered, but
176 doing so has complications when the ktime clocksource is not yet
177 the xen clocksource (ie, at boot time).
178
179 The new vcpu_op-based timer interface allows the tick timer period
180 to be changed or turned off. The tick timer is not useful as a
181 periodic timer because events are only delivered to running vcpus.
182 The one-shot timer can report when a timeout is in the past, so
183 set_next_event is capable of returning -ETIME when appropriate.
184 This interface is used when available.
185*/
186
187
188/*
189 Get a hypervisor absolute time. In theory we could maintain an
190 offset between the kernel's time and the hypervisor's time, and
191 apply that to a kernel's absolute timeout. Unfortunately the
192 hypervisor and kernel times can drift even if the kernel is using
193 the Xen clocksource, because ntp can warp the kernel's clocksource.
194*/
195static s64 get_abs_timeout(unsigned long delta)
196{
197 return xen_clocksource_read() + delta;
198}
199
200static int xen_timerop_shutdown(struct clock_event_device *evt)
201{
202 /* cancel timeout */
203 HYPERVISOR_set_timer_op(0);
204
205 return 0;
206}
207
208static int xen_timerop_set_next_event(unsigned long delta,
209 struct clock_event_device *evt)
210{
211 WARN_ON(!clockevent_state_oneshot(evt));
212
213 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
214 BUG();
215
216 /* We may have missed the deadline, but there's no real way of
217 knowing for sure. If the event was in the past, then we'll
218 get an immediate interrupt. */
219
220 return 0;
221}
222
223static struct clock_event_device xen_timerop_clockevent __ro_after_init = {
224 .name = "xen",
225 .features = CLOCK_EVT_FEAT_ONESHOT,
226
227 .max_delta_ns = 0xffffffff,
228 .max_delta_ticks = 0xffffffff,
229 .min_delta_ns = TIMER_SLOP,
230 .min_delta_ticks = TIMER_SLOP,
231
232 .mult = 1,
233 .shift = 0,
234 .rating = 500,
235
236 .set_state_shutdown = xen_timerop_shutdown,
237 .set_next_event = xen_timerop_set_next_event,
238};
239
240static int xen_vcpuop_shutdown(struct clock_event_device *evt)
241{
242 int cpu = smp_processor_id();
243
244 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, xen_vcpu_nr(cpu),
245 NULL) ||
246 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
247 NULL))
248 BUG();
249
250 return 0;
251}
252
253static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
254{
255 int cpu = smp_processor_id();
256
257 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
258 NULL))
259 BUG();
260
261 return 0;
262}
263
264static int xen_vcpuop_set_next_event(unsigned long delta,
265 struct clock_event_device *evt)
266{
267 int cpu = smp_processor_id();
268 struct vcpu_set_singleshot_timer single;
269 int ret;
270
271 WARN_ON(!clockevent_state_oneshot(evt));
272
273 single.timeout_abs_ns = get_abs_timeout(delta);
274 /* Get an event anyway, even if the timeout is already expired */
275 single.flags = 0;
276
277 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, xen_vcpu_nr(cpu),
278 &single);
279 BUG_ON(ret != 0);
280
281 return ret;
282}
283
284static struct clock_event_device xen_vcpuop_clockevent __ro_after_init = {
285 .name = "xen",
286 .features = CLOCK_EVT_FEAT_ONESHOT,
287
288 .max_delta_ns = 0xffffffff,
289 .max_delta_ticks = 0xffffffff,
290 .min_delta_ns = TIMER_SLOP,
291 .min_delta_ticks = TIMER_SLOP,
292
293 .mult = 1,
294 .shift = 0,
295 .rating = 500,
296
297 .set_state_shutdown = xen_vcpuop_shutdown,
298 .set_state_oneshot = xen_vcpuop_set_oneshot,
299 .set_next_event = xen_vcpuop_set_next_event,
300};
301
302static const struct clock_event_device *xen_clockevent =
303 &xen_timerop_clockevent;
304
305struct xen_clock_event_device {
306 struct clock_event_device evt;
307 char name[16];
308};
309static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
310
311static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
312{
313 struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
314 irqreturn_t ret;
315
316 ret = IRQ_NONE;
317 if (evt->event_handler) {
318 evt->event_handler(evt);
319 ret = IRQ_HANDLED;
320 }
321
322 return ret;
323}
324
325void xen_teardown_timer(int cpu)
326{
327 struct clock_event_device *evt;
328 evt = &per_cpu(xen_clock_events, cpu).evt;
329
330 if (evt->irq >= 0) {
331 unbind_from_irqhandler(evt->irq, NULL);
332 evt->irq = -1;
333 }
334}
335
336void xen_setup_timer(int cpu)
337{
338 struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
339 struct clock_event_device *evt = &xevt->evt;
340 int irq;
341
342 WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
343 if (evt->irq >= 0)
344 xen_teardown_timer(cpu);
345
346 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
347
348 snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);
349
350 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
351 IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER|
352 IRQF_FORCE_RESUME|IRQF_EARLY_RESUME,
353 xevt->name, NULL);
354 (void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);
355
356 memcpy(evt, xen_clockevent, sizeof(*evt));
357
358 evt->cpumask = cpumask_of(cpu);
359 evt->irq = irq;
360}
361
362
363void xen_setup_cpu_clockevents(void)
364{
365 clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
366}
367
368void xen_timer_resume(void)
369{
370 int cpu;
371
372 if (xen_clockevent != &xen_vcpuop_clockevent)
373 return;
374
375 for_each_online_cpu(cpu) {
376 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer,
377 xen_vcpu_nr(cpu), NULL))
378 BUG();
379 }
380}
381
382static const struct pv_time_ops xen_time_ops __initconst = {
383 .sched_clock = xen_sched_clock,
384 .steal_clock = xen_steal_clock,
385};
386
387static struct pvclock_vsyscall_time_info *xen_clock __read_mostly;
388static u64 xen_clock_value_saved;
389
390void xen_save_time_memory_area(void)
391{
392 struct vcpu_register_time_memory_area t;
393 int ret;
394
395 xen_clock_value_saved = xen_clocksource_read() - xen_sched_clock_offset;
396
397 if (!xen_clock)
398 return;
399
400 t.addr.v = NULL;
401
402 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
403 if (ret != 0)
404 pr_notice("Cannot save secondary vcpu_time_info (err %d)",
405 ret);
406 else
407 clear_page(xen_clock);
408}
409
410void xen_restore_time_memory_area(void)
411{
412 struct vcpu_register_time_memory_area t;
413 int ret;
414
415 if (!xen_clock)
416 goto out;
417
418 t.addr.v = &xen_clock->pvti;
419
420 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
421
422 /*
423 * We don't disable VDSO_CLOCKMODE_PVCLOCK entirely if it fails to
424 * register the secondary time info with Xen or if we migrated to a
425 * host without the necessary flags. On both of these cases what
426 * happens is either process seeing a zeroed out pvti or seeing no
427 * PVCLOCK_TSC_STABLE_BIT bit set. Userspace checks the latter and
428 * if 0, it discards the data in pvti and fallbacks to a system
429 * call for a reliable timestamp.
430 */
431 if (ret != 0)
432 pr_notice("Cannot restore secondary vcpu_time_info (err %d)",
433 ret);
434
435out:
436 /* Need pvclock_resume() before using xen_clocksource_read(). */
437 pvclock_resume();
438 xen_sched_clock_offset = xen_clocksource_read() - xen_clock_value_saved;
439}
440
441static void xen_setup_vsyscall_time_info(void)
442{
443 struct vcpu_register_time_memory_area t;
444 struct pvclock_vsyscall_time_info *ti;
445 int ret;
446
447 ti = (struct pvclock_vsyscall_time_info *)get_zeroed_page(GFP_KERNEL);
448 if (!ti)
449 return;
450
451 t.addr.v = &ti->pvti;
452
453 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
454 if (ret) {
455 pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (err %d)\n", ret);
456 free_page((unsigned long)ti);
457 return;
458 }
459
460 /*
461 * If primary time info had this bit set, secondary should too since
462 * it's the same data on both just different memory regions. But we
463 * still check it in case hypervisor is buggy.
464 */
465 if (!(ti->pvti.flags & PVCLOCK_TSC_STABLE_BIT)) {
466 t.addr.v = NULL;
467 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area,
468 0, &t);
469 if (!ret)
470 free_page((unsigned long)ti);
471
472 pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (tsc unstable)\n");
473 return;
474 }
475
476 xen_clock = ti;
477 pvclock_set_pvti_cpu0_va(xen_clock);
478
479 xen_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_PVCLOCK;
480}
481
482static void __init xen_time_init(void)
483{
484 struct pvclock_vcpu_time_info *pvti;
485 int cpu = smp_processor_id();
486 struct timespec64 tp;
487
488 /* As Dom0 is never moved, no penalty on using TSC there */
489 if (xen_initial_domain())
490 xen_clocksource.rating = 275;
491
492 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
493
494 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
495 NULL) == 0) {
496 /* Successfully turned off 100Hz tick, so we have the
497 vcpuop-based timer interface */
498 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
499 xen_clockevent = &xen_vcpuop_clockevent;
500 }
501
502 /* Set initial system time with full resolution */
503 xen_read_wallclock(&tp);
504 do_settimeofday64(&tp);
505
506 setup_force_cpu_cap(X86_FEATURE_TSC);
507
508 /*
509 * We check ahead on the primary time info if this
510 * bit is supported hence speeding up Xen clocksource.
511 */
512 pvti = &__this_cpu_read(xen_vcpu)->time;
513 if (pvti->flags & PVCLOCK_TSC_STABLE_BIT) {
514 pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
515 xen_setup_vsyscall_time_info();
516 }
517
518 xen_setup_runstate_info(cpu);
519 xen_setup_timer(cpu);
520 xen_setup_cpu_clockevents();
521
522 xen_time_setup_guest();
523
524 if (xen_initial_domain())
525 pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
526}
527
528void __init xen_init_time_ops(void)
529{
530 xen_sched_clock_offset = xen_clocksource_read();
531 pv_ops.time = xen_time_ops;
532
533 x86_init.timers.timer_init = xen_time_init;
534 x86_init.timers.setup_percpu_clockev = x86_init_noop;
535 x86_cpuinit.setup_percpu_clockev = x86_init_noop;
536
537 x86_platform.calibrate_tsc = xen_tsc_khz;
538 x86_platform.get_wallclock = xen_get_wallclock;
539 /* Dom0 uses the native method to set the hardware RTC. */
540 if (!xen_initial_domain())
541 x86_platform.set_wallclock = xen_set_wallclock;
542}
543
544#ifdef CONFIG_XEN_PVHVM
545static void xen_hvm_setup_cpu_clockevents(void)
546{
547 int cpu = smp_processor_id();
548 xen_setup_runstate_info(cpu);
549 /*
550 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
551 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
552 * early bootup and also during CPU hotplug events).
553 */
554 xen_setup_cpu_clockevents();
555}
556
557void __init xen_hvm_init_time_ops(void)
558{
559 /*
560 * vector callback is needed otherwise we cannot receive interrupts
561 * on cpu > 0 and at this point we don't know how many cpus are
562 * available.
563 */
564 if (!xen_have_vector_callback)
565 return;
566
567 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
568 pr_info("Xen doesn't support pvclock on HVM, disable pv timer");
569 return;
570 }
571
572 xen_sched_clock_offset = xen_clocksource_read();
573 pv_ops.time = xen_time_ops;
574 x86_init.timers.setup_percpu_clockev = xen_time_init;
575 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
576
577 x86_platform.calibrate_tsc = xen_tsc_khz;
578 x86_platform.get_wallclock = xen_get_wallclock;
579 x86_platform.set_wallclock = xen_set_wallclock;
580}
581#endif
582
583/* Kernel parameter to specify Xen timer slop */
584static int __init parse_xen_timer_slop(char *ptr)
585{
586 unsigned long slop = memparse(ptr, NULL);
587
588 xen_timerop_clockevent.min_delta_ns = slop;
589 xen_timerop_clockevent.min_delta_ticks = slop;
590 xen_vcpuop_clockevent.min_delta_ns = slop;
591 xen_vcpuop_clockevent.min_delta_ticks = slop;
592
593 return 0;
594}
595early_param("xen_timer_slop", parse_xen_timer_slop);