1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Clocksource driver for the synthetic counter and timers
  5 * provided by the Hyper-V hypervisor to guest VMs, as described
  6 * in the Hyper-V Top Level Functional Spec (TLFS). This driver
  7 * is instruction set architecture independent.
  8 *
  9 * Copyright (C) 2019, Microsoft, Inc.
 10 *
 11 * Author:  Michael Kelley <mikelley@microsoft.com>
 12 */
 13
 14#include <linux/percpu.h>
 15#include <linux/cpumask.h>
 16#include <linux/clockchips.h>
 17#include <linux/clocksource.h>
 18#include <linux/sched_clock.h>
 19#include <linux/mm.h>
 20#include <linux/cpuhotplug.h>
 21#include <linux/interrupt.h>
 22#include <linux/irq.h>
 23#include <linux/acpi.h>
 24#include <linux/hyperv.h>
 25#include <clocksource/hyperv_timer.h>
 26#include <asm/hyperv-tlfs.h>
 27#include <asm/mshyperv.h>
 28
 29static struct clock_event_device __percpu *hv_clock_event;
 30static u64 hv_sched_clock_offset __ro_after_init;
 31
 32/*
 33 * If false, we're using the old mechanism for stimer0 interrupts
 34 * where it sends a VMbus message when it expires. The old
 35 * mechanism is used when running on older versions of Hyper-V
 36 * that don't support Direct Mode. While Hyper-V provides
 37 * four stimer's per CPU, Linux uses only stimer0.
 38 *
 39 * Because Direct Mode does not require processing a VMbus
 40 * message, stimer interrupts can be enabled earlier in the
 41 * process of booting a CPU, and consistent with when timer
 42 * interrupts are enabled for other clocksource drivers.
 43 * However, for legacy versions of Hyper-V when Direct Mode
 44 * is not enabled, setting up stimer interrupts must be
 45 * delayed until VMbus is initialized and can process the
 46 * interrupt message.
 47 */
 48static bool direct_mode_enabled;
 49
 50static int stimer0_irq = -1;
 51static int stimer0_message_sint;
 52static DEFINE_PER_CPU(long, stimer0_evt);
 53
 54/*
 55 * Common code for stimer0 interrupts coming via Direct Mode or
 56 * as a VMbus message.
 57 */
 58void hv_stimer0_isr(void)
 59{
 60	struct clock_event_device *ce;
 61
 62	ce = this_cpu_ptr(hv_clock_event);
 63	ce->event_handler(ce);
 64}
 65EXPORT_SYMBOL_GPL(hv_stimer0_isr);
 66
 67/*
 68 * stimer0 interrupt handler for architectures that support
 69 * per-cpu interrupts, which also implies Direct Mode.
 70 */
 71static irqreturn_t hv_stimer0_percpu_isr(int irq, void *dev_id)
 72{
 73	hv_stimer0_isr();
 74	return IRQ_HANDLED;
 75}
 76
 77static int hv_ce_set_next_event(unsigned long delta,
 78				struct clock_event_device *evt)
 79{
 80	u64 current_tick;
 81
 82	current_tick = hv_read_reference_counter();
 83	current_tick += delta;
 84	hv_set_register(HV_REGISTER_STIMER0_COUNT, current_tick);
 85	return 0;
 86}
 87
 88static int hv_ce_shutdown(struct clock_event_device *evt)
 89{
 90	hv_set_register(HV_REGISTER_STIMER0_COUNT, 0);
 91	hv_set_register(HV_REGISTER_STIMER0_CONFIG, 0);
 92	if (direct_mode_enabled && stimer0_irq >= 0)
 93		disable_percpu_irq(stimer0_irq);
 94
 95	return 0;
 96}
 97
 98static int hv_ce_set_oneshot(struct clock_event_device *evt)
 99{
100	union hv_stimer_config timer_cfg;
101
102	timer_cfg.as_uint64 = 0;
103	timer_cfg.enable = 1;
104	timer_cfg.auto_enable = 1;
105	if (direct_mode_enabled) {
106		/*
107		 * When it expires, the timer will directly interrupt
108		 * on the specified hardware vector/IRQ.
109		 */
110		timer_cfg.direct_mode = 1;
111		timer_cfg.apic_vector = HYPERV_STIMER0_VECTOR;
112		if (stimer0_irq >= 0)
113			enable_percpu_irq(stimer0_irq, IRQ_TYPE_NONE);
114	} else {
115		/*
116		 * When it expires, the timer will generate a VMbus message,
117		 * to be handled by the normal VMbus interrupt handler.
118		 */
119		timer_cfg.direct_mode = 0;
120		timer_cfg.sintx = stimer0_message_sint;
121	}
122	hv_set_register(HV_REGISTER_STIMER0_CONFIG, timer_cfg.as_uint64);
123	return 0;
124}
125
126/*
127 * hv_stimer_init - Per-cpu initialization of the clockevent
128 */
129static int hv_stimer_init(unsigned int cpu)
130{
131	struct clock_event_device *ce;
132
133	if (!hv_clock_event)
134		return 0;
135
136	ce = per_cpu_ptr(hv_clock_event, cpu);
137	ce->name = "Hyper-V clockevent";
138	ce->features = CLOCK_EVT_FEAT_ONESHOT;
139	ce->cpumask = cpumask_of(cpu);
140	ce->rating = 1000;
141	ce->set_state_shutdown = hv_ce_shutdown;
142	ce->set_state_oneshot = hv_ce_set_oneshot;
143	ce->set_next_event = hv_ce_set_next_event;
144
145	clockevents_config_and_register(ce,
146					HV_CLOCK_HZ,
147					HV_MIN_DELTA_TICKS,
148					HV_MAX_MAX_DELTA_TICKS);
149	return 0;
150}
151
152/*
153 * hv_stimer_cleanup - Per-cpu cleanup of the clockevent
154 */
155int hv_stimer_cleanup(unsigned int cpu)
156{
157	struct clock_event_device *ce;
158
159	if (!hv_clock_event)
160		return 0;
161
162	/*
163	 * In the legacy case where Direct Mode is not enabled
164	 * (which can only be on x86/64), stimer cleanup happens
165	 * relatively early in the CPU offlining process. We
166	 * must unbind the stimer-based clockevent device so
167	 * that the LAPIC timer can take over until clockevents
168	 * are no longer needed in the offlining process. Note
169	 * that clockevents_unbind_device() eventually calls
170	 * hv_ce_shutdown().
171	 *
172	 * The unbind should not be done when Direct Mode is
173	 * enabled because we may be on an architecture where
174	 * there are no other clockevent devices to fallback to.
175	 */
176	ce = per_cpu_ptr(hv_clock_event, cpu);
177	if (direct_mode_enabled)
178		hv_ce_shutdown(ce);
179	else
180		clockevents_unbind_device(ce, cpu);
181
182	return 0;
183}
184EXPORT_SYMBOL_GPL(hv_stimer_cleanup);
185
186/*
187 * These placeholders are overridden by arch specific code on
188 * architectures that need special setup of the stimer0 IRQ because
189 * they don't support per-cpu IRQs (such as x86/x64).
190 */
191void __weak hv_setup_stimer0_handler(void (*handler)(void))
192{
193};
194
195void __weak hv_remove_stimer0_handler(void)
196{
197};
198
199/* Called only on architectures with per-cpu IRQs (i.e., not x86/x64) */
200static int hv_setup_stimer0_irq(void)
201{
202	int ret;
203
204	ret = acpi_register_gsi(NULL, HYPERV_STIMER0_VECTOR,
205			ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_HIGH);
206	if (ret < 0) {
207		pr_err("Can't register Hyper-V stimer0 GSI. Error %d", ret);
208		return ret;
209	}
210	stimer0_irq = ret;
211
212	ret = request_percpu_irq(stimer0_irq, hv_stimer0_percpu_isr,
213		"Hyper-V stimer0", &stimer0_evt);
214	if (ret) {
215		pr_err("Can't request Hyper-V stimer0 IRQ %d. Error %d",
216			stimer0_irq, ret);
217		acpi_unregister_gsi(stimer0_irq);
218		stimer0_irq = -1;
219	}
220	return ret;
221}
222
223static void hv_remove_stimer0_irq(void)
224{
225	if (stimer0_irq == -1) {
226		hv_remove_stimer0_handler();
227	} else {
228		free_percpu_irq(stimer0_irq, &stimer0_evt);
229		acpi_unregister_gsi(stimer0_irq);
230		stimer0_irq = -1;
231	}
232}
233
234/* hv_stimer_alloc - Global initialization of the clockevent and stimer0 */
235int hv_stimer_alloc(bool have_percpu_irqs)
236{
237	int ret;
238
239	/*
240	 * Synthetic timers are always available except on old versions of
241	 * Hyper-V on x86.  In that case, return as error as Linux will use a
242	 * clockevent based on emulated LAPIC timer hardware.
243	 */
244	if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE))
245		return -EINVAL;
246
247	hv_clock_event = alloc_percpu(struct clock_event_device);
248	if (!hv_clock_event)
249		return -ENOMEM;
250
251	direct_mode_enabled = ms_hyperv.misc_features &
252			HV_STIMER_DIRECT_MODE_AVAILABLE;
253
254	/*
255	 * If Direct Mode isn't enabled, the remainder of the initialization
256	 * is done later by hv_stimer_legacy_init()
257	 */
258	if (!direct_mode_enabled)
259		return 0;
260
261	if (have_percpu_irqs) {
262		ret = hv_setup_stimer0_irq();
263		if (ret)
264			goto free_clock_event;
265	} else {
266		hv_setup_stimer0_handler(hv_stimer0_isr);
267	}
268
269	/*
270	 * Since we are in Direct Mode, stimer initialization
271	 * can be done now with a CPUHP value in the same range
272	 * as other clockevent devices.
273	 */
274	ret = cpuhp_setup_state(CPUHP_AP_HYPERV_TIMER_STARTING,
275			"clockevents/hyperv/stimer:starting",
276			hv_stimer_init, hv_stimer_cleanup);
277	if (ret < 0) {
278		hv_remove_stimer0_irq();
279		goto free_clock_event;
280	}
281	return ret;
282
283free_clock_event:
284	free_percpu(hv_clock_event);
285	hv_clock_event = NULL;
286	return ret;
287}
288EXPORT_SYMBOL_GPL(hv_stimer_alloc);
289
290/*
291 * hv_stimer_legacy_init -- Called from the VMbus driver to handle
292 * the case when Direct Mode is not enabled, and the stimer
293 * must be initialized late in the CPU onlining process.
294 *
295 */
296void hv_stimer_legacy_init(unsigned int cpu, int sint)
297{
298	if (direct_mode_enabled)
299		return;
300
301	/*
302	 * This function gets called by each vCPU, so setting the
303	 * global stimer_message_sint value each time is conceptually
304	 * not ideal, but the value passed in is always the same and
305	 * it avoids introducing yet another interface into this
306	 * clocksource driver just to set the sint in the legacy case.
307	 */
308	stimer0_message_sint = sint;
309	(void)hv_stimer_init(cpu);
310}
311EXPORT_SYMBOL_GPL(hv_stimer_legacy_init);
312
313/*
314 * hv_stimer_legacy_cleanup -- Called from the VMbus driver to
315 * handle the case when Direct Mode is not enabled, and the
316 * stimer must be cleaned up early in the CPU offlining
317 * process.
318 */
319void hv_stimer_legacy_cleanup(unsigned int cpu)
320{
321	if (direct_mode_enabled)
322		return;
323	(void)hv_stimer_cleanup(cpu);
324}
325EXPORT_SYMBOL_GPL(hv_stimer_legacy_cleanup);
326
327/*
328 * Do a global cleanup of clockevents for the cases of kexec and
329 * vmbus exit
330 */
331void hv_stimer_global_cleanup(void)
332{
333	int	cpu;
334
335	/*
336	 * hv_stime_legacy_cleanup() will stop the stimer if Direct
337	 * Mode is not enabled, and fallback to the LAPIC timer.
338	 */
339	for_each_present_cpu(cpu) {
340		hv_stimer_legacy_cleanup(cpu);
341	}
342
343	if (!hv_clock_event)
344		return;
345
346	if (direct_mode_enabled) {
347		cpuhp_remove_state(CPUHP_AP_HYPERV_TIMER_STARTING);
348		hv_remove_stimer0_irq();
349		stimer0_irq = -1;
350	}
351	free_percpu(hv_clock_event);
352	hv_clock_event = NULL;
353
354}
355EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup);
356
357/*
358 * Code and definitions for the Hyper-V clocksources.  Two
359 * clocksources are defined: one that reads the Hyper-V defined MSR, and
360 * the other that uses the TSC reference page feature as defined in the
361 * TLFS.  The MSR version is for compatibility with old versions of
362 * Hyper-V and 32-bit x86.  The TSC reference page version is preferred.
363 */
364
365static union {
366	struct ms_hyperv_tsc_page page;
367	u8 reserved[PAGE_SIZE];
368} tsc_pg __aligned(PAGE_SIZE);
369
370static struct ms_hyperv_tsc_page *tsc_page = &tsc_pg.page;
371static unsigned long tsc_pfn;
372
373unsigned long hv_get_tsc_pfn(void)
374{
375	return tsc_pfn;
376}
377EXPORT_SYMBOL_GPL(hv_get_tsc_pfn);
378
379struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
380{
381	return tsc_page;
382}
383EXPORT_SYMBOL_GPL(hv_get_tsc_page);
384
385static u64 notrace read_hv_clock_tsc(void)
386{
387	u64 current_tick = hv_read_tsc_page(hv_get_tsc_page());
388
389	if (current_tick == U64_MAX)
390		current_tick = hv_get_register(HV_REGISTER_TIME_REF_COUNT);
391
392	return current_tick;
393}
394
395static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg)
396{
397	return read_hv_clock_tsc();
398}
399
400static u64 notrace read_hv_sched_clock_tsc(void)
401{
402	return (read_hv_clock_tsc() - hv_sched_clock_offset) *
403		(NSEC_PER_SEC / HV_CLOCK_HZ);
404}
405
406static void suspend_hv_clock_tsc(struct clocksource *arg)
407{
408	union hv_reference_tsc_msr tsc_msr;
409
410	/* Disable the TSC page */
411	tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC);
412	tsc_msr.enable = 0;
413	hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64);
414}
415
416
417static void resume_hv_clock_tsc(struct clocksource *arg)
418{
419	union hv_reference_tsc_msr tsc_msr;
420
421	/* Re-enable the TSC page */
422	tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC);
423	tsc_msr.enable = 1;
424	tsc_msr.pfn = tsc_pfn;
425	hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64);
426}
427
428#ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK
429static int hv_cs_enable(struct clocksource *cs)
430{
431	vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK);
432	return 0;
433}
434#endif
435
436static struct clocksource hyperv_cs_tsc = {
437	.name	= "hyperv_clocksource_tsc_page",
438	.rating	= 500,
439	.read	= read_hv_clock_tsc_cs,
440	.mask	= CLOCKSOURCE_MASK(64),
441	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
442	.suspend= suspend_hv_clock_tsc,
443	.resume	= resume_hv_clock_tsc,
444#ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK
445	.enable = hv_cs_enable,
446	.vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK,
447#else
448	.vdso_clock_mode = VDSO_CLOCKMODE_NONE,
449#endif
450};
451
452static u64 notrace read_hv_clock_msr(void)
453{
454	/*
455	 * Read the partition counter to get the current tick count. This count
456	 * is set to 0 when the partition is created and is incremented in
457	 * 100 nanosecond units.
458	 */
459	return hv_get_register(HV_REGISTER_TIME_REF_COUNT);
460}
461
462static u64 notrace read_hv_clock_msr_cs(struct clocksource *arg)
463{
464	return read_hv_clock_msr();
465}
466
467static u64 notrace read_hv_sched_clock_msr(void)
468{
469	return (read_hv_clock_msr() - hv_sched_clock_offset) *
470		(NSEC_PER_SEC / HV_CLOCK_HZ);
471}
472
473static struct clocksource hyperv_cs_msr = {
474	.name	= "hyperv_clocksource_msr",
475	.rating	= 500,
476	.read	= read_hv_clock_msr_cs,
477	.mask	= CLOCKSOURCE_MASK(64),
478	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
479};
480
481/*
482 * Reference to pv_ops must be inline so objtool
483 * detection of noinstr violations can work correctly.
484 */
485#ifdef CONFIG_GENERIC_SCHED_CLOCK
486static __always_inline void hv_setup_sched_clock(void *sched_clock)
487{
488	/*
489	 * We're on an architecture with generic sched clock (not x86/x64).
490	 * The Hyper-V sched clock read function returns nanoseconds, not
491	 * the normal 100ns units of the Hyper-V synthetic clock.
492	 */
493	sched_clock_register(sched_clock, 64, NSEC_PER_SEC);
494}
495#elif defined CONFIG_PARAVIRT
496static __always_inline void hv_setup_sched_clock(void *sched_clock)
497{
498	/* We're on x86/x64 *and* using PV ops */
499	paravirt_set_sched_clock(sched_clock);
500}
501#else /* !CONFIG_GENERIC_SCHED_CLOCK && !CONFIG_PARAVIRT */
502static __always_inline void hv_setup_sched_clock(void *sched_clock) {}
503#endif /* CONFIG_GENERIC_SCHED_CLOCK */
504
505static bool __init hv_init_tsc_clocksource(void)
506{
507	union hv_reference_tsc_msr tsc_msr;
508
509	if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE))
510		return false;
511
512	/*
513	 * If Hyper-V offers TSC_INVARIANT, then the virtualized TSC correctly
514	 * handles frequency and offset changes due to live migration,
515	 * pause/resume, and other VM management operations.  So lower the
516	 * Hyper-V Reference TSC rating, causing the generic TSC to be used.
517	 * TSC_INVARIANT is not offered on ARM64, so the Hyper-V Reference
518	 * TSC will be preferred over the virtualized ARM64 arch counter.
519	 * While the Hyper-V MSR clocksource won't be used since the
520	 * Reference TSC clocksource is present, change its rating as
521	 * well for consistency.
522	 */
523	if (ms_hyperv.features & HV_ACCESS_TSC_INVARIANT) {
524		hyperv_cs_tsc.rating = 250;
525		hyperv_cs_msr.rating = 250;
526	}
527
528	hv_read_reference_counter = read_hv_clock_tsc;
529
530	/*
531	 * TSC page mapping works differently in root compared to guest.
532	 * - In guest partition the guest PFN has to be passed to the
533	 *   hypervisor.
534	 * - In root partition it's other way around: it has to map the PFN
535	 *   provided by the hypervisor.
536	 *   But it can't be mapped right here as it's too early and MMU isn't
537	 *   ready yet. So, we only set the enable bit here and will remap the
538	 *   page later in hv_remap_tsc_clocksource().
539	 *
540	 * It worth mentioning, that TSC clocksource read function
541	 * (read_hv_clock_tsc) has a MSR-based fallback mechanism, used when
542	 * TSC page is zeroed (which is the case until the PFN is remapped) and
543	 * thus TSC clocksource will work even without the real TSC page
544	 * mapped.
545	 */
546	tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC);
547	if (hv_root_partition)
548		tsc_pfn = tsc_msr.pfn;
549	else
550		tsc_pfn = HVPFN_DOWN(virt_to_phys(tsc_page));
551	tsc_msr.enable = 1;
552	tsc_msr.pfn = tsc_pfn;
553	hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64);
554
555	clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100);
556
557	hv_sched_clock_offset = hv_read_reference_counter();
558	hv_setup_sched_clock(read_hv_sched_clock_tsc);
559
560	return true;
561}
562
563void __init hv_init_clocksource(void)
564{
565	/*
566	 * Try to set up the TSC page clocksource. If it succeeds, we're
567	 * done. Otherwise, set up the MSR clocksource.  At least one of
568	 * these will always be available except on very old versions of
569	 * Hyper-V on x86.  In that case we won't have a Hyper-V
570	 * clocksource, but Linux will still run with a clocksource based
571	 * on the emulated PIT or LAPIC timer.
572	 */
573	if (hv_init_tsc_clocksource())
574		return;
575
576	if (!(ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE))
577		return;
578
579	hv_read_reference_counter = read_hv_clock_msr;
580	clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100);
581
582	hv_sched_clock_offset = hv_read_reference_counter();
583	hv_setup_sched_clock(read_hv_sched_clock_msr);
584}
585
586void __init hv_remap_tsc_clocksource(void)
587{
588	if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE))
589		return;
590
591	if (!hv_root_partition) {
592		WARN(1, "%s: attempt to remap TSC page in guest partition\n",
593		     __func__);
594		return;
595	}
596
597	tsc_page = memremap(tsc_pfn << HV_HYP_PAGE_SHIFT, sizeof(tsc_pg),
598			    MEMREMAP_WB);
599	if (!tsc_page)
600		pr_err("Failed to remap Hyper-V TSC page.\n");
601}