Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Xen time implementation.
4 *
5 * This is implemented in terms of a clocksource driver which uses
6 * the hypervisor clock as a nanosecond timebase, and a clockevent
7 * driver which uses the hypervisor's timer mechanism.
8 *
9 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
10 */
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/clocksource.h>
14#include <linux/clockchips.h>
15#include <linux/gfp.h>
16#include <linux/slab.h>
17#include <linux/pvclock_gtod.h>
18#include <linux/timekeeper_internal.h>
19
20#include <asm/pvclock.h>
21#include <asm/xen/hypervisor.h>
22#include <asm/xen/hypercall.h>
23
24#include <xen/events.h>
25#include <xen/features.h>
26#include <xen/interface/xen.h>
27#include <xen/interface/vcpu.h>
28
29#include "xen-ops.h"
30
31/* Minimum amount of time until next clock event fires */
32#define TIMER_SLOP 100000
33
34static u64 xen_sched_clock_offset __read_mostly;
35
36/* Get the TSC speed from Xen */
37static unsigned long xen_tsc_khz(void)
38{
39 struct pvclock_vcpu_time_info *info =
40 &HYPERVISOR_shared_info->vcpu_info[0].time;
41
42 setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
43 return pvclock_tsc_khz(info);
44}
45
46static u64 xen_clocksource_read(void)
47{
48 struct pvclock_vcpu_time_info *src;
49 u64 ret;
50
51 preempt_disable_notrace();
52 src = &__this_cpu_read(xen_vcpu)->time;
53 ret = pvclock_clocksource_read(src);
54 preempt_enable_notrace();
55 return ret;
56}
57
58static u64 xen_clocksource_get_cycles(struct clocksource *cs)
59{
60 return xen_clocksource_read();
61}
62
63static u64 xen_sched_clock(void)
64{
65 return xen_clocksource_read() - xen_sched_clock_offset;
66}
67
68static void xen_read_wallclock(struct timespec64 *ts)
69{
70 struct shared_info *s = HYPERVISOR_shared_info;
71 struct pvclock_wall_clock *wall_clock = &(s->wc);
72 struct pvclock_vcpu_time_info *vcpu_time;
73
74 vcpu_time = &get_cpu_var(xen_vcpu)->time;
75 pvclock_read_wallclock(wall_clock, vcpu_time, ts);
76 put_cpu_var(xen_vcpu);
77}
78
79static void xen_get_wallclock(struct timespec64 *now)
80{
81 xen_read_wallclock(now);
82}
83
84static int xen_set_wallclock(const struct timespec64 *now)
85{
86 return -ENODEV;
87}
88
89static int xen_pvclock_gtod_notify(struct notifier_block *nb,
90 unsigned long was_set, void *priv)
91{
92 /* Protected by the calling core code serialization */
93 static struct timespec64 next_sync;
94
95 struct xen_platform_op op;
96 struct timespec64 now;
97 struct timekeeper *tk = priv;
98 static bool settime64_supported = true;
99 int ret;
100
101 now.tv_sec = tk->xtime_sec;
102 now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
103
104 /*
105 * We only take the expensive HV call when the clock was set
106 * or when the 11 minutes RTC synchronization time elapsed.
107 */
108 if (!was_set && timespec64_compare(&now, &next_sync) < 0)
109 return NOTIFY_OK;
110
111again:
112 if (settime64_supported) {
113 op.cmd = XENPF_settime64;
114 op.u.settime64.mbz = 0;
115 op.u.settime64.secs = now.tv_sec;
116 op.u.settime64.nsecs = now.tv_nsec;
117 op.u.settime64.system_time = xen_clocksource_read();
118 } else {
119 op.cmd = XENPF_settime32;
120 op.u.settime32.secs = now.tv_sec;
121 op.u.settime32.nsecs = now.tv_nsec;
122 op.u.settime32.system_time = xen_clocksource_read();
123 }
124
125 ret = HYPERVISOR_platform_op(&op);
126
127 if (ret == -ENOSYS && settime64_supported) {
128 settime64_supported = false;
129 goto again;
130 }
131 if (ret < 0)
132 return NOTIFY_BAD;
133
134 /*
135 * Move the next drift compensation time 11 minutes
136 * ahead. That's emulating the sync_cmos_clock() update for
137 * the hardware RTC.
138 */
139 next_sync = now;
140 next_sync.tv_sec += 11 * 60;
141
142 return NOTIFY_OK;
143}
144
145static struct notifier_block xen_pvclock_gtod_notifier = {
146 .notifier_call = xen_pvclock_gtod_notify,
147};
148
149static int xen_cs_enable(struct clocksource *cs)
150{
151 vclocks_set_used(VDSO_CLOCKMODE_PVCLOCK);
152 return 0;
153}
154
155static struct clocksource xen_clocksource __read_mostly = {
156 .name = "xen",
157 .rating = 400,
158 .read = xen_clocksource_get_cycles,
159 .mask = CLOCKSOURCE_MASK(64),
160 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
161 .enable = xen_cs_enable,
162};
163
164/*
165 Xen clockevent implementation
166
167 Xen has two clockevent implementations:
168
169 The old timer_op one works with all released versions of Xen prior
170 to version 3.0.4. This version of the hypervisor provides a
171 single-shot timer with nanosecond resolution. However, sharing the
172 same event channel is a 100Hz tick which is delivered while the
173 vcpu is running. We don't care about or use this tick, but it will
174 cause the core time code to think the timer fired too soon, and
175 will end up resetting it each time. It could be filtered, but
176 doing so has complications when the ktime clocksource is not yet
177 the xen clocksource (ie, at boot time).
178
179 The new vcpu_op-based timer interface allows the tick timer period
180 to be changed or turned off. The tick timer is not useful as a
181 periodic timer because events are only delivered to running vcpus.
182 The one-shot timer can report when a timeout is in the past, so
183 set_next_event is capable of returning -ETIME when appropriate.
184 This interface is used when available.
185*/
186
187
188/*
189 Get a hypervisor absolute time. In theory we could maintain an
190 offset between the kernel's time and the hypervisor's time, and
191 apply that to a kernel's absolute timeout. Unfortunately the
192 hypervisor and kernel times can drift even if the kernel is using
193 the Xen clocksource, because ntp can warp the kernel's clocksource.
194*/
195static s64 get_abs_timeout(unsigned long delta)
196{
197 return xen_clocksource_read() + delta;
198}
199
200static int xen_timerop_shutdown(struct clock_event_device *evt)
201{
202 /* cancel timeout */
203 HYPERVISOR_set_timer_op(0);
204
205 return 0;
206}
207
208static int xen_timerop_set_next_event(unsigned long delta,
209 struct clock_event_device *evt)
210{
211 WARN_ON(!clockevent_state_oneshot(evt));
212
213 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
214 BUG();
215
216 /* We may have missed the deadline, but there's no real way of
217 knowing for sure. If the event was in the past, then we'll
218 get an immediate interrupt. */
219
220 return 0;
221}
222
223static struct clock_event_device xen_timerop_clockevent __ro_after_init = {
224 .name = "xen",
225 .features = CLOCK_EVT_FEAT_ONESHOT,
226
227 .max_delta_ns = 0xffffffff,
228 .max_delta_ticks = 0xffffffff,
229 .min_delta_ns = TIMER_SLOP,
230 .min_delta_ticks = TIMER_SLOP,
231
232 .mult = 1,
233 .shift = 0,
234 .rating = 500,
235
236 .set_state_shutdown = xen_timerop_shutdown,
237 .set_next_event = xen_timerop_set_next_event,
238};
239
240static int xen_vcpuop_shutdown(struct clock_event_device *evt)
241{
242 int cpu = smp_processor_id();
243
244 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, xen_vcpu_nr(cpu),
245 NULL) ||
246 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
247 NULL))
248 BUG();
249
250 return 0;
251}
252
253static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
254{
255 int cpu = smp_processor_id();
256
257 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
258 NULL))
259 BUG();
260
261 return 0;
262}
263
264static int xen_vcpuop_set_next_event(unsigned long delta,
265 struct clock_event_device *evt)
266{
267 int cpu = smp_processor_id();
268 struct vcpu_set_singleshot_timer single;
269 int ret;
270
271 WARN_ON(!clockevent_state_oneshot(evt));
272
273 single.timeout_abs_ns = get_abs_timeout(delta);
274 /* Get an event anyway, even if the timeout is already expired */
275 single.flags = 0;
276
277 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, xen_vcpu_nr(cpu),
278 &single);
279 BUG_ON(ret != 0);
280
281 return ret;
282}
283
284static struct clock_event_device xen_vcpuop_clockevent __ro_after_init = {
285 .name = "xen",
286 .features = CLOCK_EVT_FEAT_ONESHOT,
287
288 .max_delta_ns = 0xffffffff,
289 .max_delta_ticks = 0xffffffff,
290 .min_delta_ns = TIMER_SLOP,
291 .min_delta_ticks = TIMER_SLOP,
292
293 .mult = 1,
294 .shift = 0,
295 .rating = 500,
296
297 .set_state_shutdown = xen_vcpuop_shutdown,
298 .set_state_oneshot = xen_vcpuop_set_oneshot,
299 .set_next_event = xen_vcpuop_set_next_event,
300};
301
302static const struct clock_event_device *xen_clockevent =
303 &xen_timerop_clockevent;
304
305struct xen_clock_event_device {
306 struct clock_event_device evt;
307 char name[16];
308};
309static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
310
311static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
312{
313 struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
314 irqreturn_t ret;
315
316 ret = IRQ_NONE;
317 if (evt->event_handler) {
318 evt->event_handler(evt);
319 ret = IRQ_HANDLED;
320 }
321
322 return ret;
323}
324
325void xen_teardown_timer(int cpu)
326{
327 struct clock_event_device *evt;
328 evt = &per_cpu(xen_clock_events, cpu).evt;
329
330 if (evt->irq >= 0) {
331 unbind_from_irqhandler(evt->irq, NULL);
332 evt->irq = -1;
333 }
334}
335
336void xen_setup_timer(int cpu)
337{
338 struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
339 struct clock_event_device *evt = &xevt->evt;
340 int irq;
341
342 WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
343 if (evt->irq >= 0)
344 xen_teardown_timer(cpu);
345
346 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
347
348 snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);
349
350 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
351 IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER|
352 IRQF_FORCE_RESUME|IRQF_EARLY_RESUME,
353 xevt->name, NULL);
354 (void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);
355
356 memcpy(evt, xen_clockevent, sizeof(*evt));
357
358 evt->cpumask = cpumask_of(cpu);
359 evt->irq = irq;
360}
361
362
363void xen_setup_cpu_clockevents(void)
364{
365 clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
366}
367
368void xen_timer_resume(void)
369{
370 int cpu;
371
372 if (xen_clockevent != &xen_vcpuop_clockevent)
373 return;
374
375 for_each_online_cpu(cpu) {
376 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer,
377 xen_vcpu_nr(cpu), NULL))
378 BUG();
379 }
380}
381
382static struct pvclock_vsyscall_time_info *xen_clock __read_mostly;
383static u64 xen_clock_value_saved;
384
385void xen_save_time_memory_area(void)
386{
387 struct vcpu_register_time_memory_area t;
388 int ret;
389
390 xen_clock_value_saved = xen_clocksource_read() - xen_sched_clock_offset;
391
392 if (!xen_clock)
393 return;
394
395 t.addr.v = NULL;
396
397 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
398 if (ret != 0)
399 pr_notice("Cannot save secondary vcpu_time_info (err %d)",
400 ret);
401 else
402 clear_page(xen_clock);
403}
404
405void xen_restore_time_memory_area(void)
406{
407 struct vcpu_register_time_memory_area t;
408 int ret;
409
410 if (!xen_clock)
411 goto out;
412
413 t.addr.v = &xen_clock->pvti;
414
415 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
416
417 /*
418 * We don't disable VDSO_CLOCKMODE_PVCLOCK entirely if it fails to
419 * register the secondary time info with Xen or if we migrated to a
420 * host without the necessary flags. On both of these cases what
421 * happens is either process seeing a zeroed out pvti or seeing no
422 * PVCLOCK_TSC_STABLE_BIT bit set. Userspace checks the latter and
423 * if 0, it discards the data in pvti and fallbacks to a system
424 * call for a reliable timestamp.
425 */
426 if (ret != 0)
427 pr_notice("Cannot restore secondary vcpu_time_info (err %d)",
428 ret);
429
430out:
431 /* Need pvclock_resume() before using xen_clocksource_read(). */
432 pvclock_resume();
433 xen_sched_clock_offset = xen_clocksource_read() - xen_clock_value_saved;
434}
435
436static void xen_setup_vsyscall_time_info(void)
437{
438 struct vcpu_register_time_memory_area t;
439 struct pvclock_vsyscall_time_info *ti;
440 int ret;
441
442 ti = (struct pvclock_vsyscall_time_info *)get_zeroed_page(GFP_KERNEL);
443 if (!ti)
444 return;
445
446 t.addr.v = &ti->pvti;
447
448 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
449 if (ret) {
450 pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (err %d)\n", ret);
451 free_page((unsigned long)ti);
452 return;
453 }
454
455 /*
456 * If primary time info had this bit set, secondary should too since
457 * it's the same data on both just different memory regions. But we
458 * still check it in case hypervisor is buggy.
459 */
460 if (!(ti->pvti.flags & PVCLOCK_TSC_STABLE_BIT)) {
461 t.addr.v = NULL;
462 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area,
463 0, &t);
464 if (!ret)
465 free_page((unsigned long)ti);
466
467 pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (tsc unstable)\n");
468 return;
469 }
470
471 xen_clock = ti;
472 pvclock_set_pvti_cpu0_va(xen_clock);
473
474 xen_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_PVCLOCK;
475}
476
477static void __init xen_time_init(void)
478{
479 struct pvclock_vcpu_time_info *pvti;
480 int cpu = smp_processor_id();
481 struct timespec64 tp;
482
483 /* As Dom0 is never moved, no penalty on using TSC there */
484 if (xen_initial_domain())
485 xen_clocksource.rating = 275;
486
487 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
488
489 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
490 NULL) == 0) {
491 /* Successfully turned off 100Hz tick, so we have the
492 vcpuop-based timer interface */
493 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
494 xen_clockevent = &xen_vcpuop_clockevent;
495 }
496
497 /* Set initial system time with full resolution */
498 xen_read_wallclock(&tp);
499 do_settimeofday64(&tp);
500
501 setup_force_cpu_cap(X86_FEATURE_TSC);
502
503 /*
504 * We check ahead on the primary time info if this
505 * bit is supported hence speeding up Xen clocksource.
506 */
507 pvti = &__this_cpu_read(xen_vcpu)->time;
508 if (pvti->flags & PVCLOCK_TSC_STABLE_BIT) {
509 pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
510 xen_setup_vsyscall_time_info();
511 }
512
513 xen_setup_runstate_info(cpu);
514 xen_setup_timer(cpu);
515 xen_setup_cpu_clockevents();
516
517 xen_time_setup_guest();
518
519 if (xen_initial_domain())
520 pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
521}
522
523static void __init xen_init_time_common(void)
524{
525 xen_sched_clock_offset = xen_clocksource_read();
526 static_call_update(pv_steal_clock, xen_steal_clock);
527 paravirt_set_sched_clock(xen_sched_clock);
528
529 x86_platform.calibrate_tsc = xen_tsc_khz;
530 x86_platform.get_wallclock = xen_get_wallclock;
531}
532
533void __init xen_init_time_ops(void)
534{
535 xen_init_time_common();
536
537 x86_init.timers.timer_init = xen_time_init;
538 x86_init.timers.setup_percpu_clockev = x86_init_noop;
539 x86_cpuinit.setup_percpu_clockev = x86_init_noop;
540
541 /* Dom0 uses the native method to set the hardware RTC. */
542 if (!xen_initial_domain())
543 x86_platform.set_wallclock = xen_set_wallclock;
544}
545
546#ifdef CONFIG_XEN_PVHVM
547static void xen_hvm_setup_cpu_clockevents(void)
548{
549 int cpu = smp_processor_id();
550 xen_setup_runstate_info(cpu);
551 /*
552 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
553 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
554 * early bootup and also during CPU hotplug events).
555 */
556 xen_setup_cpu_clockevents();
557}
558
559void __init xen_hvm_init_time_ops(void)
560{
561 static bool hvm_time_initialized;
562
563 if (hvm_time_initialized)
564 return;
565
566 /*
567 * vector callback is needed otherwise we cannot receive interrupts
568 * on cpu > 0 and at this point we don't know how many cpus are
569 * available.
570 */
571 if (!xen_have_vector_callback)
572 return;
573
574 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
575 pr_info_once("Xen doesn't support pvclock on HVM, disable pv timer");
576 return;
577 }
578
579 /*
580 * Only MAX_VIRT_CPUS 'vcpu_info' are embedded inside 'shared_info'.
581 * The __this_cpu_read(xen_vcpu) is still NULL when Xen HVM guest
582 * boots on vcpu >= MAX_VIRT_CPUS (e.g., kexec), To access
583 * __this_cpu_read(xen_vcpu) via xen_clocksource_read() will panic.
584 *
585 * The xen_hvm_init_time_ops() should be called again later after
586 * __this_cpu_read(xen_vcpu) is available.
587 */
588 if (!__this_cpu_read(xen_vcpu)) {
589 pr_info("Delay xen_init_time_common() as kernel is running on vcpu=%d\n",
590 xen_vcpu_nr(0));
591 return;
592 }
593
594 xen_init_time_common();
595
596 x86_init.timers.setup_percpu_clockev = xen_time_init;
597 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
598
599 x86_platform.set_wallclock = xen_set_wallclock;
600
601 hvm_time_initialized = true;
602}
603#endif
604
605/* Kernel parameter to specify Xen timer slop */
606static int __init parse_xen_timer_slop(char *ptr)
607{
608 unsigned long slop = memparse(ptr, NULL);
609
610 xen_timerop_clockevent.min_delta_ns = slop;
611 xen_timerop_clockevent.min_delta_ticks = slop;
612 xen_vcpuop_clockevent.min_delta_ns = slop;
613 xen_vcpuop_clockevent.min_delta_ticks = slop;
614
615 return 0;
616}
617early_param("xen_timer_slop", parse_xen_timer_slop);
1/*
2 * Xen time implementation.
3 *
4 * This is implemented in terms of a clocksource driver which uses
5 * the hypervisor clock as a nanosecond timebase, and a clockevent
6 * driver which uses the hypervisor's timer mechanism.
7 *
8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
9 */
10#include <linux/kernel.h>
11#include <linux/interrupt.h>
12#include <linux/clocksource.h>
13#include <linux/clockchips.h>
14#include <linux/kernel_stat.h>
15#include <linux/math64.h>
16#include <linux/gfp.h>
17#include <linux/slab.h>
18#include <linux/pvclock_gtod.h>
19#include <linux/timekeeper_internal.h>
20
21#include <asm/pvclock.h>
22#include <asm/xen/hypervisor.h>
23#include <asm/xen/hypercall.h>
24
25#include <xen/events.h>
26#include <xen/features.h>
27#include <xen/interface/xen.h>
28#include <xen/interface/vcpu.h>
29
30#include "xen-ops.h"
31
32/* Xen may fire a timer up to this many ns early */
33#define TIMER_SLOP 100000
34#define NS_PER_TICK (1000000000LL / HZ)
35
36/* snapshots of runstate info */
37static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot);
38
39/* unused ns of stolen time */
40static DEFINE_PER_CPU(u64, xen_residual_stolen);
41
42static void do_stolen_accounting(void)
43{
44 struct vcpu_runstate_info state;
45 struct vcpu_runstate_info *snap;
46 s64 runnable, offline, stolen;
47 cputime_t ticks;
48
49 xen_get_runstate_snapshot(&state);
50
51 WARN_ON(state.state != RUNSTATE_running);
52
53 snap = this_cpu_ptr(&xen_runstate_snapshot);
54
55 /* work out how much time the VCPU has not been runn*ing* */
56 runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
57 offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
58
59 *snap = state;
60
61 /* Add the appropriate number of ticks of stolen time,
62 including any left-overs from last time. */
63 stolen = runnable + offline + __this_cpu_read(xen_residual_stolen);
64
65 if (stolen < 0)
66 stolen = 0;
67
68 ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
69 __this_cpu_write(xen_residual_stolen, stolen);
70 account_steal_ticks(ticks);
71}
72
73/* Get the TSC speed from Xen */
74static unsigned long xen_tsc_khz(void)
75{
76 struct pvclock_vcpu_time_info *info =
77 &HYPERVISOR_shared_info->vcpu_info[0].time;
78
79 return pvclock_tsc_khz(info);
80}
81
82cycle_t xen_clocksource_read(void)
83{
84 struct pvclock_vcpu_time_info *src;
85 cycle_t ret;
86
87 preempt_disable_notrace();
88 src = &__this_cpu_read(xen_vcpu)->time;
89 ret = pvclock_clocksource_read(src);
90 preempt_enable_notrace();
91 return ret;
92}
93
94static cycle_t xen_clocksource_get_cycles(struct clocksource *cs)
95{
96 return xen_clocksource_read();
97}
98
99static void xen_read_wallclock(struct timespec *ts)
100{
101 struct shared_info *s = HYPERVISOR_shared_info;
102 struct pvclock_wall_clock *wall_clock = &(s->wc);
103 struct pvclock_vcpu_time_info *vcpu_time;
104
105 vcpu_time = &get_cpu_var(xen_vcpu)->time;
106 pvclock_read_wallclock(wall_clock, vcpu_time, ts);
107 put_cpu_var(xen_vcpu);
108}
109
110static void xen_get_wallclock(struct timespec *now)
111{
112 xen_read_wallclock(now);
113}
114
115static int xen_set_wallclock(const struct timespec *now)
116{
117 return -1;
118}
119
120static int xen_pvclock_gtod_notify(struct notifier_block *nb,
121 unsigned long was_set, void *priv)
122{
123 /* Protected by the calling core code serialization */
124 static struct timespec64 next_sync;
125
126 struct xen_platform_op op;
127 struct timespec64 now;
128 struct timekeeper *tk = priv;
129 static bool settime64_supported = true;
130 int ret;
131
132 now.tv_sec = tk->xtime_sec;
133 now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
134
135 /*
136 * We only take the expensive HV call when the clock was set
137 * or when the 11 minutes RTC synchronization time elapsed.
138 */
139 if (!was_set && timespec64_compare(&now, &next_sync) < 0)
140 return NOTIFY_OK;
141
142again:
143 if (settime64_supported) {
144 op.cmd = XENPF_settime64;
145 op.u.settime64.mbz = 0;
146 op.u.settime64.secs = now.tv_sec;
147 op.u.settime64.nsecs = now.tv_nsec;
148 op.u.settime64.system_time = xen_clocksource_read();
149 } else {
150 op.cmd = XENPF_settime32;
151 op.u.settime32.secs = now.tv_sec;
152 op.u.settime32.nsecs = now.tv_nsec;
153 op.u.settime32.system_time = xen_clocksource_read();
154 }
155
156 ret = HYPERVISOR_platform_op(&op);
157
158 if (ret == -ENOSYS && settime64_supported) {
159 settime64_supported = false;
160 goto again;
161 }
162 if (ret < 0)
163 return NOTIFY_BAD;
164
165 /*
166 * Move the next drift compensation time 11 minutes
167 * ahead. That's emulating the sync_cmos_clock() update for
168 * the hardware RTC.
169 */
170 next_sync = now;
171 next_sync.tv_sec += 11 * 60;
172
173 return NOTIFY_OK;
174}
175
176static struct notifier_block xen_pvclock_gtod_notifier = {
177 .notifier_call = xen_pvclock_gtod_notify,
178};
179
180static struct clocksource xen_clocksource __read_mostly = {
181 .name = "xen",
182 .rating = 400,
183 .read = xen_clocksource_get_cycles,
184 .mask = ~0,
185 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
186};
187
188/*
189 Xen clockevent implementation
190
191 Xen has two clockevent implementations:
192
193 The old timer_op one works with all released versions of Xen prior
194 to version 3.0.4. This version of the hypervisor provides a
195 single-shot timer with nanosecond resolution. However, sharing the
196 same event channel is a 100Hz tick which is delivered while the
197 vcpu is running. We don't care about or use this tick, but it will
198 cause the core time code to think the timer fired too soon, and
199 will end up resetting it each time. It could be filtered, but
200 doing so has complications when the ktime clocksource is not yet
201 the xen clocksource (ie, at boot time).
202
203 The new vcpu_op-based timer interface allows the tick timer period
204 to be changed or turned off. The tick timer is not useful as a
205 periodic timer because events are only delivered to running vcpus.
206 The one-shot timer can report when a timeout is in the past, so
207 set_next_event is capable of returning -ETIME when appropriate.
208 This interface is used when available.
209*/
210
211
212/*
213 Get a hypervisor absolute time. In theory we could maintain an
214 offset between the kernel's time and the hypervisor's time, and
215 apply that to a kernel's absolute timeout. Unfortunately the
216 hypervisor and kernel times can drift even if the kernel is using
217 the Xen clocksource, because ntp can warp the kernel's clocksource.
218*/
219static s64 get_abs_timeout(unsigned long delta)
220{
221 return xen_clocksource_read() + delta;
222}
223
224static int xen_timerop_shutdown(struct clock_event_device *evt)
225{
226 /* cancel timeout */
227 HYPERVISOR_set_timer_op(0);
228
229 return 0;
230}
231
232static int xen_timerop_set_next_event(unsigned long delta,
233 struct clock_event_device *evt)
234{
235 WARN_ON(!clockevent_state_oneshot(evt));
236
237 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
238 BUG();
239
240 /* We may have missed the deadline, but there's no real way of
241 knowing for sure. If the event was in the past, then we'll
242 get an immediate interrupt. */
243
244 return 0;
245}
246
247static const struct clock_event_device xen_timerop_clockevent = {
248 .name = "xen",
249 .features = CLOCK_EVT_FEAT_ONESHOT,
250
251 .max_delta_ns = 0xffffffff,
252 .min_delta_ns = TIMER_SLOP,
253
254 .mult = 1,
255 .shift = 0,
256 .rating = 500,
257
258 .set_state_shutdown = xen_timerop_shutdown,
259 .set_next_event = xen_timerop_set_next_event,
260};
261
262static int xen_vcpuop_shutdown(struct clock_event_device *evt)
263{
264 int cpu = smp_processor_id();
265
266 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
267 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
268 BUG();
269
270 return 0;
271}
272
273static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
274{
275 int cpu = smp_processor_id();
276
277 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
278 BUG();
279
280 return 0;
281}
282
283static int xen_vcpuop_set_next_event(unsigned long delta,
284 struct clock_event_device *evt)
285{
286 int cpu = smp_processor_id();
287 struct vcpu_set_singleshot_timer single;
288 int ret;
289
290 WARN_ON(!clockevent_state_oneshot(evt));
291
292 single.timeout_abs_ns = get_abs_timeout(delta);
293 single.flags = VCPU_SSHOTTMR_future;
294
295 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
296
297 BUG_ON(ret != 0 && ret != -ETIME);
298
299 return ret;
300}
301
302static const struct clock_event_device xen_vcpuop_clockevent = {
303 .name = "xen",
304 .features = CLOCK_EVT_FEAT_ONESHOT,
305
306 .max_delta_ns = 0xffffffff,
307 .min_delta_ns = TIMER_SLOP,
308
309 .mult = 1,
310 .shift = 0,
311 .rating = 500,
312
313 .set_state_shutdown = xen_vcpuop_shutdown,
314 .set_state_oneshot = xen_vcpuop_set_oneshot,
315 .set_next_event = xen_vcpuop_set_next_event,
316};
317
318static const struct clock_event_device *xen_clockevent =
319 &xen_timerop_clockevent;
320
321struct xen_clock_event_device {
322 struct clock_event_device evt;
323 char name[16];
324};
325static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
326
327static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
328{
329 struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
330 irqreturn_t ret;
331
332 ret = IRQ_NONE;
333 if (evt->event_handler) {
334 evt->event_handler(evt);
335 ret = IRQ_HANDLED;
336 }
337
338 do_stolen_accounting();
339
340 return ret;
341}
342
343void xen_teardown_timer(int cpu)
344{
345 struct clock_event_device *evt;
346 BUG_ON(cpu == 0);
347 evt = &per_cpu(xen_clock_events, cpu).evt;
348
349 if (evt->irq >= 0) {
350 unbind_from_irqhandler(evt->irq, NULL);
351 evt->irq = -1;
352 }
353}
354
355void xen_setup_timer(int cpu)
356{
357 struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
358 struct clock_event_device *evt = &xevt->evt;
359 int irq;
360
361 WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
362 if (evt->irq >= 0)
363 xen_teardown_timer(cpu);
364
365 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
366
367 snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);
368
369 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
370 IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER|
371 IRQF_FORCE_RESUME|IRQF_EARLY_RESUME,
372 xevt->name, NULL);
373 (void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);
374
375 memcpy(evt, xen_clockevent, sizeof(*evt));
376
377 evt->cpumask = cpumask_of(cpu);
378 evt->irq = irq;
379}
380
381
382void xen_setup_cpu_clockevents(void)
383{
384 clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
385}
386
387void xen_timer_resume(void)
388{
389 int cpu;
390
391 pvclock_resume();
392
393 if (xen_clockevent != &xen_vcpuop_clockevent)
394 return;
395
396 for_each_online_cpu(cpu) {
397 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
398 BUG();
399 }
400}
401
402static const struct pv_time_ops xen_time_ops __initconst = {
403 .sched_clock = xen_clocksource_read,
404};
405
406static void __init xen_time_init(void)
407{
408 int cpu = smp_processor_id();
409 struct timespec tp;
410
411 /* As Dom0 is never moved, no penalty on using TSC there */
412 if (xen_initial_domain())
413 xen_clocksource.rating = 275;
414
415 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
416
417 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
418 /* Successfully turned off 100Hz tick, so we have the
419 vcpuop-based timer interface */
420 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
421 xen_clockevent = &xen_vcpuop_clockevent;
422 }
423
424 /* Set initial system time with full resolution */
425 xen_read_wallclock(&tp);
426 do_settimeofday(&tp);
427
428 setup_force_cpu_cap(X86_FEATURE_TSC);
429
430 xen_setup_runstate_info(cpu);
431 xen_setup_timer(cpu);
432 xen_setup_cpu_clockevents();
433
434 if (xen_initial_domain())
435 pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
436}
437
438void __init xen_init_time_ops(void)
439{
440 pv_time_ops = xen_time_ops;
441
442 x86_init.timers.timer_init = xen_time_init;
443 x86_init.timers.setup_percpu_clockev = x86_init_noop;
444 x86_cpuinit.setup_percpu_clockev = x86_init_noop;
445
446 x86_platform.calibrate_tsc = xen_tsc_khz;
447 x86_platform.get_wallclock = xen_get_wallclock;
448 /* Dom0 uses the native method to set the hardware RTC. */
449 if (!xen_initial_domain())
450 x86_platform.set_wallclock = xen_set_wallclock;
451}
452
453#ifdef CONFIG_XEN_PVHVM
454static void xen_hvm_setup_cpu_clockevents(void)
455{
456 int cpu = smp_processor_id();
457 xen_setup_runstate_info(cpu);
458 /*
459 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
460 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
461 * early bootup and also during CPU hotplug events).
462 */
463 xen_setup_cpu_clockevents();
464}
465
466void __init xen_hvm_init_time_ops(void)
467{
468 /* vector callback is needed otherwise we cannot receive interrupts
469 * on cpu > 0 and at this point we don't know how many cpus are
470 * available */
471 if (!xen_have_vector_callback)
472 return;
473 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
474 printk(KERN_INFO "Xen doesn't support pvclock on HVM,"
475 "disable pv timer\n");
476 return;
477 }
478
479 pv_time_ops = xen_time_ops;
480 x86_init.timers.setup_percpu_clockev = xen_time_init;
481 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
482
483 x86_platform.calibrate_tsc = xen_tsc_khz;
484 x86_platform.get_wallclock = xen_get_wallclock;
485 x86_platform.set_wallclock = xen_set_wallclock;
486}
487#endif