Loading...
1/*
2 * Xen time implementation.
3 *
4 * This is implemented in terms of a clocksource driver which uses
5 * the hypervisor clock as a nanosecond timebase, and a clockevent
6 * driver which uses the hypervisor's timer mechanism.
7 *
8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
9 */
10#include <linux/kernel.h>
11#include <linux/interrupt.h>
12#include <linux/clocksource.h>
13#include <linux/clockchips.h>
14#include <linux/kernel_stat.h>
15#include <linux/math64.h>
16#include <linux/gfp.h>
17
18#include <asm/pvclock.h>
19#include <asm/xen/hypervisor.h>
20#include <asm/xen/hypercall.h>
21
22#include <xen/events.h>
23#include <xen/features.h>
24#include <xen/interface/xen.h>
25#include <xen/interface/vcpu.h>
26
27#include "xen-ops.h"
28
29/* Xen may fire a timer up to this many ns early */
30#define TIMER_SLOP 100000
31#define NS_PER_TICK (1000000000LL / HZ)
32
33/* runstate info updated by Xen */
34static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
35
36/* snapshots of runstate info */
37static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot);
38
39/* unused ns of stolen and blocked time */
40static DEFINE_PER_CPU(u64, xen_residual_stolen);
41static DEFINE_PER_CPU(u64, xen_residual_blocked);
42
43/* return an consistent snapshot of 64-bit time/counter value */
44static u64 get64(const u64 *p)
45{
46 u64 ret;
47
48 if (BITS_PER_LONG < 64) {
49 u32 *p32 = (u32 *)p;
50 u32 h, l;
51
52 /*
53 * Read high then low, and then make sure high is
54 * still the same; this will only loop if low wraps
55 * and carries into high.
56 * XXX some clean way to make this endian-proof?
57 */
58 do {
59 h = p32[1];
60 barrier();
61 l = p32[0];
62 barrier();
63 } while (p32[1] != h);
64
65 ret = (((u64)h) << 32) | l;
66 } else
67 ret = *p;
68
69 return ret;
70}
71
72/*
73 * Runstate accounting
74 */
75static void get_runstate_snapshot(struct vcpu_runstate_info *res)
76{
77 u64 state_time;
78 struct vcpu_runstate_info *state;
79
80 BUG_ON(preemptible());
81
82 state = &__get_cpu_var(xen_runstate);
83
84 /*
85 * The runstate info is always updated by the hypervisor on
86 * the current CPU, so there's no need to use anything
87 * stronger than a compiler barrier when fetching it.
88 */
89 do {
90 state_time = get64(&state->state_entry_time);
91 barrier();
92 *res = *state;
93 barrier();
94 } while (get64(&state->state_entry_time) != state_time);
95}
96
97/* return true when a vcpu could run but has no real cpu to run on */
98bool xen_vcpu_stolen(int vcpu)
99{
100 return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
101}
102
103void xen_setup_runstate_info(int cpu)
104{
105 struct vcpu_register_runstate_memory_area area;
106
107 area.addr.v = &per_cpu(xen_runstate, cpu);
108
109 if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
110 cpu, &area))
111 BUG();
112}
113
114static void do_stolen_accounting(void)
115{
116 struct vcpu_runstate_info state;
117 struct vcpu_runstate_info *snap;
118 s64 blocked, runnable, offline, stolen;
119 cputime_t ticks;
120
121 get_runstate_snapshot(&state);
122
123 WARN_ON(state.state != RUNSTATE_running);
124
125 snap = &__get_cpu_var(xen_runstate_snapshot);
126
127 /* work out how much time the VCPU has not been runn*ing* */
128 blocked = state.time[RUNSTATE_blocked] - snap->time[RUNSTATE_blocked];
129 runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
130 offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
131
132 *snap = state;
133
134 /* Add the appropriate number of ticks of stolen time,
135 including any left-overs from last time. */
136 stolen = runnable + offline + __this_cpu_read(xen_residual_stolen);
137
138 if (stolen < 0)
139 stolen = 0;
140
141 ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
142 __this_cpu_write(xen_residual_stolen, stolen);
143 account_steal_ticks(ticks);
144
145 /* Add the appropriate number of ticks of blocked time,
146 including any left-overs from last time. */
147 blocked += __this_cpu_read(xen_residual_blocked);
148
149 if (blocked < 0)
150 blocked = 0;
151
152 ticks = iter_div_u64_rem(blocked, NS_PER_TICK, &blocked);
153 __this_cpu_write(xen_residual_blocked, blocked);
154 account_idle_ticks(ticks);
155}
156
157/* Get the TSC speed from Xen */
158static unsigned long xen_tsc_khz(void)
159{
160 struct pvclock_vcpu_time_info *info =
161 &HYPERVISOR_shared_info->vcpu_info[0].time;
162
163 return pvclock_tsc_khz(info);
164}
165
166cycle_t xen_clocksource_read(void)
167{
168 struct pvclock_vcpu_time_info *src;
169 cycle_t ret;
170
171 preempt_disable_notrace();
172 src = &__get_cpu_var(xen_vcpu)->time;
173 ret = pvclock_clocksource_read(src);
174 preempt_enable_notrace();
175 return ret;
176}
177
178static cycle_t xen_clocksource_get_cycles(struct clocksource *cs)
179{
180 return xen_clocksource_read();
181}
182
183static void xen_read_wallclock(struct timespec *ts)
184{
185 struct shared_info *s = HYPERVISOR_shared_info;
186 struct pvclock_wall_clock *wall_clock = &(s->wc);
187 struct pvclock_vcpu_time_info *vcpu_time;
188
189 vcpu_time = &get_cpu_var(xen_vcpu)->time;
190 pvclock_read_wallclock(wall_clock, vcpu_time, ts);
191 put_cpu_var(xen_vcpu);
192}
193
194static unsigned long xen_get_wallclock(void)
195{
196 struct timespec ts;
197
198 xen_read_wallclock(&ts);
199 return ts.tv_sec;
200}
201
202static int xen_set_wallclock(unsigned long now)
203{
204 struct xen_platform_op op;
205 int rc;
206
207 /* do nothing for domU */
208 if (!xen_initial_domain())
209 return -1;
210
211 op.cmd = XENPF_settime;
212 op.u.settime.secs = now;
213 op.u.settime.nsecs = 0;
214 op.u.settime.system_time = xen_clocksource_read();
215
216 rc = HYPERVISOR_dom0_op(&op);
217 WARN(rc != 0, "XENPF_settime failed: now=%ld\n", now);
218
219 return rc;
220}
221
222static struct clocksource xen_clocksource __read_mostly = {
223 .name = "xen",
224 .rating = 400,
225 .read = xen_clocksource_get_cycles,
226 .mask = ~0,
227 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
228};
229
230/*
231 Xen clockevent implementation
232
233 Xen has two clockevent implementations:
234
235 The old timer_op one works with all released versions of Xen prior
236 to version 3.0.4. This version of the hypervisor provides a
237 single-shot timer with nanosecond resolution. However, sharing the
238 same event channel is a 100Hz tick which is delivered while the
239 vcpu is running. We don't care about or use this tick, but it will
240 cause the core time code to think the timer fired too soon, and
241 will end up resetting it each time. It could be filtered, but
242 doing so has complications when the ktime clocksource is not yet
243 the xen clocksource (ie, at boot time).
244
245 The new vcpu_op-based timer interface allows the tick timer period
246 to be changed or turned off. The tick timer is not useful as a
247 periodic timer because events are only delivered to running vcpus.
248 The one-shot timer can report when a timeout is in the past, so
249 set_next_event is capable of returning -ETIME when appropriate.
250 This interface is used when available.
251*/
252
253
254/*
255 Get a hypervisor absolute time. In theory we could maintain an
256 offset between the kernel's time and the hypervisor's time, and
257 apply that to a kernel's absolute timeout. Unfortunately the
258 hypervisor and kernel times can drift even if the kernel is using
259 the Xen clocksource, because ntp can warp the kernel's clocksource.
260*/
261static s64 get_abs_timeout(unsigned long delta)
262{
263 return xen_clocksource_read() + delta;
264}
265
266static void xen_timerop_set_mode(enum clock_event_mode mode,
267 struct clock_event_device *evt)
268{
269 switch (mode) {
270 case CLOCK_EVT_MODE_PERIODIC:
271 /* unsupported */
272 WARN_ON(1);
273 break;
274
275 case CLOCK_EVT_MODE_ONESHOT:
276 case CLOCK_EVT_MODE_RESUME:
277 break;
278
279 case CLOCK_EVT_MODE_UNUSED:
280 case CLOCK_EVT_MODE_SHUTDOWN:
281 HYPERVISOR_set_timer_op(0); /* cancel timeout */
282 break;
283 }
284}
285
286static int xen_timerop_set_next_event(unsigned long delta,
287 struct clock_event_device *evt)
288{
289 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
290
291 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
292 BUG();
293
294 /* We may have missed the deadline, but there's no real way of
295 knowing for sure. If the event was in the past, then we'll
296 get an immediate interrupt. */
297
298 return 0;
299}
300
301static const struct clock_event_device xen_timerop_clockevent = {
302 .name = "xen",
303 .features = CLOCK_EVT_FEAT_ONESHOT,
304
305 .max_delta_ns = 0xffffffff,
306 .min_delta_ns = TIMER_SLOP,
307
308 .mult = 1,
309 .shift = 0,
310 .rating = 500,
311
312 .set_mode = xen_timerop_set_mode,
313 .set_next_event = xen_timerop_set_next_event,
314};
315
316
317
318static void xen_vcpuop_set_mode(enum clock_event_mode mode,
319 struct clock_event_device *evt)
320{
321 int cpu = smp_processor_id();
322
323 switch (mode) {
324 case CLOCK_EVT_MODE_PERIODIC:
325 WARN_ON(1); /* unsupported */
326 break;
327
328 case CLOCK_EVT_MODE_ONESHOT:
329 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
330 BUG();
331 break;
332
333 case CLOCK_EVT_MODE_UNUSED:
334 case CLOCK_EVT_MODE_SHUTDOWN:
335 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
336 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
337 BUG();
338 break;
339 case CLOCK_EVT_MODE_RESUME:
340 break;
341 }
342}
343
344static int xen_vcpuop_set_next_event(unsigned long delta,
345 struct clock_event_device *evt)
346{
347 int cpu = smp_processor_id();
348 struct vcpu_set_singleshot_timer single;
349 int ret;
350
351 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
352
353 single.timeout_abs_ns = get_abs_timeout(delta);
354 single.flags = VCPU_SSHOTTMR_future;
355
356 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
357
358 BUG_ON(ret != 0 && ret != -ETIME);
359
360 return ret;
361}
362
363static const struct clock_event_device xen_vcpuop_clockevent = {
364 .name = "xen",
365 .features = CLOCK_EVT_FEAT_ONESHOT,
366
367 .max_delta_ns = 0xffffffff,
368 .min_delta_ns = TIMER_SLOP,
369
370 .mult = 1,
371 .shift = 0,
372 .rating = 500,
373
374 .set_mode = xen_vcpuop_set_mode,
375 .set_next_event = xen_vcpuop_set_next_event,
376};
377
378static const struct clock_event_device *xen_clockevent =
379 &xen_timerop_clockevent;
380static DEFINE_PER_CPU(struct clock_event_device, xen_clock_events);
381
382static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
383{
384 struct clock_event_device *evt = &__get_cpu_var(xen_clock_events);
385 irqreturn_t ret;
386
387 ret = IRQ_NONE;
388 if (evt->event_handler) {
389 evt->event_handler(evt);
390 ret = IRQ_HANDLED;
391 }
392
393 do_stolen_accounting();
394
395 return ret;
396}
397
398void xen_setup_timer(int cpu)
399{
400 const char *name;
401 struct clock_event_device *evt;
402 int irq;
403
404 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
405
406 name = kasprintf(GFP_KERNEL, "timer%d", cpu);
407 if (!name)
408 name = "<timer kasprintf failed>";
409
410 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
411 IRQF_DISABLED|IRQF_PERCPU|
412 IRQF_NOBALANCING|IRQF_TIMER|
413 IRQF_FORCE_RESUME,
414 name, NULL);
415
416 evt = &per_cpu(xen_clock_events, cpu);
417 memcpy(evt, xen_clockevent, sizeof(*evt));
418
419 evt->cpumask = cpumask_of(cpu);
420 evt->irq = irq;
421}
422
423void xen_teardown_timer(int cpu)
424{
425 struct clock_event_device *evt;
426 BUG_ON(cpu == 0);
427 evt = &per_cpu(xen_clock_events, cpu);
428 unbind_from_irqhandler(evt->irq, NULL);
429}
430
431void xen_setup_cpu_clockevents(void)
432{
433 BUG_ON(preemptible());
434
435 clockevents_register_device(&__get_cpu_var(xen_clock_events));
436}
437
438void xen_timer_resume(void)
439{
440 int cpu;
441
442 pvclock_resume();
443
444 if (xen_clockevent != &xen_vcpuop_clockevent)
445 return;
446
447 for_each_online_cpu(cpu) {
448 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
449 BUG();
450 }
451}
452
453static const struct pv_time_ops xen_time_ops __initconst = {
454 .sched_clock = xen_clocksource_read,
455};
456
457static void __init xen_time_init(void)
458{
459 int cpu = smp_processor_id();
460 struct timespec tp;
461
462 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
463
464 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
465 /* Successfully turned off 100Hz tick, so we have the
466 vcpuop-based timer interface */
467 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
468 xen_clockevent = &xen_vcpuop_clockevent;
469 }
470
471 /* Set initial system time with full resolution */
472 xen_read_wallclock(&tp);
473 do_settimeofday(&tp);
474
475 setup_force_cpu_cap(X86_FEATURE_TSC);
476
477 xen_setup_runstate_info(cpu);
478 xen_setup_timer(cpu);
479 xen_setup_cpu_clockevents();
480}
481
482void __init xen_init_time_ops(void)
483{
484 pv_time_ops = xen_time_ops;
485
486 x86_init.timers.timer_init = xen_time_init;
487 x86_init.timers.setup_percpu_clockev = x86_init_noop;
488 x86_cpuinit.setup_percpu_clockev = x86_init_noop;
489
490 x86_platform.calibrate_tsc = xen_tsc_khz;
491 x86_platform.get_wallclock = xen_get_wallclock;
492 x86_platform.set_wallclock = xen_set_wallclock;
493}
494
495#ifdef CONFIG_XEN_PVHVM
496static void xen_hvm_setup_cpu_clockevents(void)
497{
498 int cpu = smp_processor_id();
499 xen_setup_runstate_info(cpu);
500 xen_setup_timer(cpu);
501 xen_setup_cpu_clockevents();
502}
503
504void __init xen_hvm_init_time_ops(void)
505{
506 /* vector callback is needed otherwise we cannot receive interrupts
507 * on cpu > 0 and at this point we don't know how many cpus are
508 * available */
509 if (!xen_have_vector_callback)
510 return;
511 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
512 printk(KERN_INFO "Xen doesn't support pvclock on HVM,"
513 "disable pv timer\n");
514 return;
515 }
516
517 pv_time_ops = xen_time_ops;
518 x86_init.timers.setup_percpu_clockev = xen_time_init;
519 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
520
521 x86_platform.calibrate_tsc = xen_tsc_khz;
522 x86_platform.get_wallclock = xen_get_wallclock;
523 x86_platform.set_wallclock = xen_set_wallclock;
524}
525#endif
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Xen time implementation.
4 *
5 * This is implemented in terms of a clocksource driver which uses
6 * the hypervisor clock as a nanosecond timebase, and a clockevent
7 * driver which uses the hypervisor's timer mechanism.
8 *
9 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
10 */
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/clocksource.h>
14#include <linux/clockchips.h>
15#include <linux/gfp.h>
16#include <linux/slab.h>
17#include <linux/pvclock_gtod.h>
18#include <linux/timekeeper_internal.h>
19
20#include <asm/pvclock.h>
21#include <asm/xen/hypervisor.h>
22#include <asm/xen/hypercall.h>
23
24#include <xen/events.h>
25#include <xen/features.h>
26#include <xen/interface/xen.h>
27#include <xen/interface/vcpu.h>
28
29#include "xen-ops.h"
30
31/* Minimum amount of time until next clock event fires */
32#define TIMER_SLOP 100000
33
34static u64 xen_sched_clock_offset __read_mostly;
35
36/* Get the TSC speed from Xen */
37static unsigned long xen_tsc_khz(void)
38{
39 struct pvclock_vcpu_time_info *info =
40 &HYPERVISOR_shared_info->vcpu_info[0].time;
41
42 return pvclock_tsc_khz(info);
43}
44
45static u64 xen_clocksource_read(void)
46{
47 struct pvclock_vcpu_time_info *src;
48 u64 ret;
49
50 preempt_disable_notrace();
51 src = &__this_cpu_read(xen_vcpu)->time;
52 ret = pvclock_clocksource_read(src);
53 preempt_enable_notrace();
54 return ret;
55}
56
57static u64 xen_clocksource_get_cycles(struct clocksource *cs)
58{
59 return xen_clocksource_read();
60}
61
62static u64 xen_sched_clock(void)
63{
64 return xen_clocksource_read() - xen_sched_clock_offset;
65}
66
67static void xen_read_wallclock(struct timespec64 *ts)
68{
69 struct shared_info *s = HYPERVISOR_shared_info;
70 struct pvclock_wall_clock *wall_clock = &(s->wc);
71 struct pvclock_vcpu_time_info *vcpu_time;
72
73 vcpu_time = &get_cpu_var(xen_vcpu)->time;
74 pvclock_read_wallclock(wall_clock, vcpu_time, ts);
75 put_cpu_var(xen_vcpu);
76}
77
78static void xen_get_wallclock(struct timespec64 *now)
79{
80 xen_read_wallclock(now);
81}
82
83static int xen_set_wallclock(const struct timespec64 *now)
84{
85 return -ENODEV;
86}
87
88static int xen_pvclock_gtod_notify(struct notifier_block *nb,
89 unsigned long was_set, void *priv)
90{
91 /* Protected by the calling core code serialization */
92 static struct timespec64 next_sync;
93
94 struct xen_platform_op op;
95 struct timespec64 now;
96 struct timekeeper *tk = priv;
97 static bool settime64_supported = true;
98 int ret;
99
100 now.tv_sec = tk->xtime_sec;
101 now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
102
103 /*
104 * We only take the expensive HV call when the clock was set
105 * or when the 11 minutes RTC synchronization time elapsed.
106 */
107 if (!was_set && timespec64_compare(&now, &next_sync) < 0)
108 return NOTIFY_OK;
109
110again:
111 if (settime64_supported) {
112 op.cmd = XENPF_settime64;
113 op.u.settime64.mbz = 0;
114 op.u.settime64.secs = now.tv_sec;
115 op.u.settime64.nsecs = now.tv_nsec;
116 op.u.settime64.system_time = xen_clocksource_read();
117 } else {
118 op.cmd = XENPF_settime32;
119 op.u.settime32.secs = now.tv_sec;
120 op.u.settime32.nsecs = now.tv_nsec;
121 op.u.settime32.system_time = xen_clocksource_read();
122 }
123
124 ret = HYPERVISOR_platform_op(&op);
125
126 if (ret == -ENOSYS && settime64_supported) {
127 settime64_supported = false;
128 goto again;
129 }
130 if (ret < 0)
131 return NOTIFY_BAD;
132
133 /*
134 * Move the next drift compensation time 11 minutes
135 * ahead. That's emulating the sync_cmos_clock() update for
136 * the hardware RTC.
137 */
138 next_sync = now;
139 next_sync.tv_sec += 11 * 60;
140
141 return NOTIFY_OK;
142}
143
144static struct notifier_block xen_pvclock_gtod_notifier = {
145 .notifier_call = xen_pvclock_gtod_notify,
146};
147
148static struct clocksource xen_clocksource __read_mostly = {
149 .name = "xen",
150 .rating = 400,
151 .read = xen_clocksource_get_cycles,
152 .mask = ~0,
153 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
154};
155
156/*
157 Xen clockevent implementation
158
159 Xen has two clockevent implementations:
160
161 The old timer_op one works with all released versions of Xen prior
162 to version 3.0.4. This version of the hypervisor provides a
163 single-shot timer with nanosecond resolution. However, sharing the
164 same event channel is a 100Hz tick which is delivered while the
165 vcpu is running. We don't care about or use this tick, but it will
166 cause the core time code to think the timer fired too soon, and
167 will end up resetting it each time. It could be filtered, but
168 doing so has complications when the ktime clocksource is not yet
169 the xen clocksource (ie, at boot time).
170
171 The new vcpu_op-based timer interface allows the tick timer period
172 to be changed or turned off. The tick timer is not useful as a
173 periodic timer because events are only delivered to running vcpus.
174 The one-shot timer can report when a timeout is in the past, so
175 set_next_event is capable of returning -ETIME when appropriate.
176 This interface is used when available.
177*/
178
179
180/*
181 Get a hypervisor absolute time. In theory we could maintain an
182 offset between the kernel's time and the hypervisor's time, and
183 apply that to a kernel's absolute timeout. Unfortunately the
184 hypervisor and kernel times can drift even if the kernel is using
185 the Xen clocksource, because ntp can warp the kernel's clocksource.
186*/
187static s64 get_abs_timeout(unsigned long delta)
188{
189 return xen_clocksource_read() + delta;
190}
191
192static int xen_timerop_shutdown(struct clock_event_device *evt)
193{
194 /* cancel timeout */
195 HYPERVISOR_set_timer_op(0);
196
197 return 0;
198}
199
200static int xen_timerop_set_next_event(unsigned long delta,
201 struct clock_event_device *evt)
202{
203 WARN_ON(!clockevent_state_oneshot(evt));
204
205 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
206 BUG();
207
208 /* We may have missed the deadline, but there's no real way of
209 knowing for sure. If the event was in the past, then we'll
210 get an immediate interrupt. */
211
212 return 0;
213}
214
215static struct clock_event_device xen_timerop_clockevent __ro_after_init = {
216 .name = "xen",
217 .features = CLOCK_EVT_FEAT_ONESHOT,
218
219 .max_delta_ns = 0xffffffff,
220 .max_delta_ticks = 0xffffffff,
221 .min_delta_ns = TIMER_SLOP,
222 .min_delta_ticks = TIMER_SLOP,
223
224 .mult = 1,
225 .shift = 0,
226 .rating = 500,
227
228 .set_state_shutdown = xen_timerop_shutdown,
229 .set_next_event = xen_timerop_set_next_event,
230};
231
232static int xen_vcpuop_shutdown(struct clock_event_device *evt)
233{
234 int cpu = smp_processor_id();
235
236 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, xen_vcpu_nr(cpu),
237 NULL) ||
238 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
239 NULL))
240 BUG();
241
242 return 0;
243}
244
245static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
246{
247 int cpu = smp_processor_id();
248
249 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
250 NULL))
251 BUG();
252
253 return 0;
254}
255
256static int xen_vcpuop_set_next_event(unsigned long delta,
257 struct clock_event_device *evt)
258{
259 int cpu = smp_processor_id();
260 struct vcpu_set_singleshot_timer single;
261 int ret;
262
263 WARN_ON(!clockevent_state_oneshot(evt));
264
265 single.timeout_abs_ns = get_abs_timeout(delta);
266 /* Get an event anyway, even if the timeout is already expired */
267 single.flags = 0;
268
269 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, xen_vcpu_nr(cpu),
270 &single);
271 BUG_ON(ret != 0);
272
273 return ret;
274}
275
276static struct clock_event_device xen_vcpuop_clockevent __ro_after_init = {
277 .name = "xen",
278 .features = CLOCK_EVT_FEAT_ONESHOT,
279
280 .max_delta_ns = 0xffffffff,
281 .max_delta_ticks = 0xffffffff,
282 .min_delta_ns = TIMER_SLOP,
283 .min_delta_ticks = TIMER_SLOP,
284
285 .mult = 1,
286 .shift = 0,
287 .rating = 500,
288
289 .set_state_shutdown = xen_vcpuop_shutdown,
290 .set_state_oneshot = xen_vcpuop_set_oneshot,
291 .set_next_event = xen_vcpuop_set_next_event,
292};
293
294static const struct clock_event_device *xen_clockevent =
295 &xen_timerop_clockevent;
296
297struct xen_clock_event_device {
298 struct clock_event_device evt;
299 char name[16];
300};
301static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
302
303static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
304{
305 struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
306 irqreturn_t ret;
307
308 ret = IRQ_NONE;
309 if (evt->event_handler) {
310 evt->event_handler(evt);
311 ret = IRQ_HANDLED;
312 }
313
314 return ret;
315}
316
317void xen_teardown_timer(int cpu)
318{
319 struct clock_event_device *evt;
320 evt = &per_cpu(xen_clock_events, cpu).evt;
321
322 if (evt->irq >= 0) {
323 unbind_from_irqhandler(evt->irq, NULL);
324 evt->irq = -1;
325 }
326}
327
328void xen_setup_timer(int cpu)
329{
330 struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
331 struct clock_event_device *evt = &xevt->evt;
332 int irq;
333
334 WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
335 if (evt->irq >= 0)
336 xen_teardown_timer(cpu);
337
338 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
339
340 snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);
341
342 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
343 IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER|
344 IRQF_FORCE_RESUME|IRQF_EARLY_RESUME,
345 xevt->name, NULL);
346 (void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);
347
348 memcpy(evt, xen_clockevent, sizeof(*evt));
349
350 evt->cpumask = cpumask_of(cpu);
351 evt->irq = irq;
352}
353
354
355void xen_setup_cpu_clockevents(void)
356{
357 clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
358}
359
360void xen_timer_resume(void)
361{
362 int cpu;
363
364 if (xen_clockevent != &xen_vcpuop_clockevent)
365 return;
366
367 for_each_online_cpu(cpu) {
368 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer,
369 xen_vcpu_nr(cpu), NULL))
370 BUG();
371 }
372}
373
374static const struct pv_time_ops xen_time_ops __initconst = {
375 .sched_clock = xen_sched_clock,
376 .steal_clock = xen_steal_clock,
377};
378
379static struct pvclock_vsyscall_time_info *xen_clock __read_mostly;
380static u64 xen_clock_value_saved;
381
382void xen_save_time_memory_area(void)
383{
384 struct vcpu_register_time_memory_area t;
385 int ret;
386
387 xen_clock_value_saved = xen_clocksource_read() - xen_sched_clock_offset;
388
389 if (!xen_clock)
390 return;
391
392 t.addr.v = NULL;
393
394 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
395 if (ret != 0)
396 pr_notice("Cannot save secondary vcpu_time_info (err %d)",
397 ret);
398 else
399 clear_page(xen_clock);
400}
401
402void xen_restore_time_memory_area(void)
403{
404 struct vcpu_register_time_memory_area t;
405 int ret;
406
407 if (!xen_clock)
408 goto out;
409
410 t.addr.v = &xen_clock->pvti;
411
412 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
413
414 /*
415 * We don't disable VCLOCK_PVCLOCK entirely if it fails to register the
416 * secondary time info with Xen or if we migrated to a host without the
417 * necessary flags. On both of these cases what happens is either
418 * process seeing a zeroed out pvti or seeing no PVCLOCK_TSC_STABLE_BIT
419 * bit set. Userspace checks the latter and if 0, it discards the data
420 * in pvti and fallbacks to a system call for a reliable timestamp.
421 */
422 if (ret != 0)
423 pr_notice("Cannot restore secondary vcpu_time_info (err %d)",
424 ret);
425
426out:
427 /* Need pvclock_resume() before using xen_clocksource_read(). */
428 pvclock_resume();
429 xen_sched_clock_offset = xen_clocksource_read() - xen_clock_value_saved;
430}
431
432static void xen_setup_vsyscall_time_info(void)
433{
434 struct vcpu_register_time_memory_area t;
435 struct pvclock_vsyscall_time_info *ti;
436 int ret;
437
438 ti = (struct pvclock_vsyscall_time_info *)get_zeroed_page(GFP_KERNEL);
439 if (!ti)
440 return;
441
442 t.addr.v = &ti->pvti;
443
444 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
445 if (ret) {
446 pr_notice("xen: VCLOCK_PVCLOCK not supported (err %d)\n", ret);
447 free_page((unsigned long)ti);
448 return;
449 }
450
451 /*
452 * If primary time info had this bit set, secondary should too since
453 * it's the same data on both just different memory regions. But we
454 * still check it in case hypervisor is buggy.
455 */
456 if (!(ti->pvti.flags & PVCLOCK_TSC_STABLE_BIT)) {
457 t.addr.v = NULL;
458 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area,
459 0, &t);
460 if (!ret)
461 free_page((unsigned long)ti);
462
463 pr_notice("xen: VCLOCK_PVCLOCK not supported (tsc unstable)\n");
464 return;
465 }
466
467 xen_clock = ti;
468 pvclock_set_pvti_cpu0_va(xen_clock);
469
470 xen_clocksource.archdata.vclock_mode = VCLOCK_PVCLOCK;
471}
472
473static void __init xen_time_init(void)
474{
475 struct pvclock_vcpu_time_info *pvti;
476 int cpu = smp_processor_id();
477 struct timespec64 tp;
478
479 /* As Dom0 is never moved, no penalty on using TSC there */
480 if (xen_initial_domain())
481 xen_clocksource.rating = 275;
482
483 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
484
485 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
486 NULL) == 0) {
487 /* Successfully turned off 100Hz tick, so we have the
488 vcpuop-based timer interface */
489 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
490 xen_clockevent = &xen_vcpuop_clockevent;
491 }
492
493 /* Set initial system time with full resolution */
494 xen_read_wallclock(&tp);
495 do_settimeofday64(&tp);
496
497 setup_force_cpu_cap(X86_FEATURE_TSC);
498
499 /*
500 * We check ahead on the primary time info if this
501 * bit is supported hence speeding up Xen clocksource.
502 */
503 pvti = &__this_cpu_read(xen_vcpu)->time;
504 if (pvti->flags & PVCLOCK_TSC_STABLE_BIT) {
505 pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
506 xen_setup_vsyscall_time_info();
507 }
508
509 xen_setup_runstate_info(cpu);
510 xen_setup_timer(cpu);
511 xen_setup_cpu_clockevents();
512
513 xen_time_setup_guest();
514
515 if (xen_initial_domain())
516 pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
517}
518
519void __init xen_init_time_ops(void)
520{
521 xen_sched_clock_offset = xen_clocksource_read();
522 pv_ops.time = xen_time_ops;
523
524 x86_init.timers.timer_init = xen_time_init;
525 x86_init.timers.setup_percpu_clockev = x86_init_noop;
526 x86_cpuinit.setup_percpu_clockev = x86_init_noop;
527
528 x86_platform.calibrate_tsc = xen_tsc_khz;
529 x86_platform.get_wallclock = xen_get_wallclock;
530 /* Dom0 uses the native method to set the hardware RTC. */
531 if (!xen_initial_domain())
532 x86_platform.set_wallclock = xen_set_wallclock;
533}
534
535#ifdef CONFIG_XEN_PVHVM
536static void xen_hvm_setup_cpu_clockevents(void)
537{
538 int cpu = smp_processor_id();
539 xen_setup_runstate_info(cpu);
540 /*
541 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
542 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
543 * early bootup and also during CPU hotplug events).
544 */
545 xen_setup_cpu_clockevents();
546}
547
548void __init xen_hvm_init_time_ops(void)
549{
550 /*
551 * vector callback is needed otherwise we cannot receive interrupts
552 * on cpu > 0 and at this point we don't know how many cpus are
553 * available.
554 */
555 if (!xen_have_vector_callback)
556 return;
557
558 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
559 pr_info("Xen doesn't support pvclock on HVM, disable pv timer");
560 return;
561 }
562
563 xen_sched_clock_offset = xen_clocksource_read();
564 pv_ops.time = xen_time_ops;
565 x86_init.timers.setup_percpu_clockev = xen_time_init;
566 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
567
568 x86_platform.calibrate_tsc = xen_tsc_khz;
569 x86_platform.get_wallclock = xen_get_wallclock;
570 x86_platform.set_wallclock = xen_set_wallclock;
571}
572#endif
573
574/* Kernel parameter to specify Xen timer slop */
575static int __init parse_xen_timer_slop(char *ptr)
576{
577 unsigned long slop = memparse(ptr, NULL);
578
579 xen_timerop_clockevent.min_delta_ns = slop;
580 xen_timerop_clockevent.min_delta_ticks = slop;
581 xen_vcpuop_clockevent.min_delta_ns = slop;
582 xen_vcpuop_clockevent.min_delta_ticks = slop;
583
584 return 0;
585}
586early_param("xen_timer_slop", parse_xen_timer_slop);