1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/drivers/clocksource/arm_arch_timer.c
   4 *
   5 *  Copyright (C) 2011 ARM Ltd.
   6 *  All Rights Reserved
   7 */
   8
   9#define pr_fmt(fmt) 	"arch_timer: " fmt
  10
  11#include <linux/init.h>
  12#include <linux/kernel.h>
  13#include <linux/device.h>
  14#include <linux/smp.h>
  15#include <linux/cpu.h>
  16#include <linux/cpu_pm.h>
  17#include <linux/clockchips.h>
  18#include <linux/clocksource.h>
  19#include <linux/clocksource_ids.h>
  20#include <linux/interrupt.h>
  21#include <linux/kstrtox.h>
  22#include <linux/of_irq.h>
  23#include <linux/of_address.h>
  24#include <linux/io.h>
  25#include <linux/slab.h>
  26#include <linux/sched/clock.h>
  27#include <linux/sched_clock.h>
  28#include <linux/acpi.h>
  29#include <linux/arm-smccc.h>
  30#include <linux/ptp_kvm.h>
  31
  32#include <asm/arch_timer.h>
  33#include <asm/virt.h>
  34
  35#include <clocksource/arm_arch_timer.h>
  36
  37#define CNTTIDR		0x08
  38#define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4))
  39
  40#define CNTACR(n)	(0x40 + ((n) * 4))
  41#define CNTACR_RPCT	BIT(0)
  42#define CNTACR_RVCT	BIT(1)
  43#define CNTACR_RFRQ	BIT(2)
  44#define CNTACR_RVOFF	BIT(3)
  45#define CNTACR_RWVT	BIT(4)
  46#define CNTACR_RWPT	BIT(5)
  47
  48#define CNTPCT_LO	0x00
  49#define CNTVCT_LO	0x08
 
  50#define CNTFRQ		0x10
  51#define CNTP_CVAL_LO	0x20
  52#define CNTP_CTL	0x2c
  53#define CNTV_CVAL_LO	0x30
  54#define CNTV_CTL	0x3c
  55
  56/*
  57 * The minimum amount of time a generic counter is guaranteed to not roll over
  58 * (40 years)
  59 */
  60#define MIN_ROLLOVER_SECS	(40ULL * 365 * 24 * 3600)
  61
  62static unsigned arch_timers_present __initdata;
  63
 
 
  64struct arch_timer {
  65	void __iomem *base;
  66	struct clock_event_device evt;
  67};
  68
  69static struct arch_timer *arch_timer_mem __ro_after_init;
  70
  71#define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  72
  73static u32 arch_timer_rate __ro_after_init;
  74static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI] __ro_after_init;
  75
  76static const char *arch_timer_ppi_names[ARCH_TIMER_MAX_TIMER_PPI] = {
  77	[ARCH_TIMER_PHYS_SECURE_PPI]	= "sec-phys",
  78	[ARCH_TIMER_PHYS_NONSECURE_PPI]	= "phys",
  79	[ARCH_TIMER_VIRT_PPI]		= "virt",
  80	[ARCH_TIMER_HYP_PPI]		= "hyp-phys",
  81	[ARCH_TIMER_HYP_VIRT_PPI]	= "hyp-virt",
  82};
  83
  84static struct clock_event_device __percpu *arch_timer_evt;
  85
  86static enum arch_timer_ppi_nr arch_timer_uses_ppi __ro_after_init = ARCH_TIMER_VIRT_PPI;
  87static bool arch_timer_c3stop __ro_after_init;
  88static bool arch_timer_mem_use_virtual __ro_after_init;
  89static bool arch_counter_suspend_stop __ro_after_init;
  90#ifdef CONFIG_GENERIC_GETTIMEOFDAY
  91static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;
  92#else
  93static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;
  94#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
  95
  96static cpumask_t evtstrm_available = CPU_MASK_NONE;
  97static bool evtstrm_enable __ro_after_init = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  98
  99static int __init early_evtstrm_cfg(char *buf)
 100{
 101	return kstrtobool(buf, &evtstrm_enable);
 102}
 103early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
 104
 105/*
 106 * Makes an educated guess at a valid counter width based on the Generic Timer
 107 * specification. Of note:
 108 *   1) the system counter is at least 56 bits wide
 109 *   2) a roll-over time of not less than 40 years
 110 *
 111 * See 'ARM DDI 0487G.a D11.1.2 ("The system counter")' for more details.
 112 */
 113static int arch_counter_get_width(void)
 114{
 115	u64 min_cycles = MIN_ROLLOVER_SECS * arch_timer_rate;
 116
 117	/* guarantee the returned width is within the valid range */
 118	return clamp_val(ilog2(min_cycles - 1) + 1, 56, 64);
 119}
 120
 121/*
 122 * Architected system timer support.
 123 */
 124
 125static __always_inline
 126void arch_timer_reg_write(int access, enum arch_timer_reg reg, u64 val,
 127			  struct clock_event_device *clk)
 128{
 129	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 130		struct arch_timer *timer = to_arch_timer(clk);
 131		switch (reg) {
 132		case ARCH_TIMER_REG_CTRL:
 133			writel_relaxed((u32)val, timer->base + CNTP_CTL);
 134			break;
 135		case ARCH_TIMER_REG_CVAL:
 136			/*
 137			 * Not guaranteed to be atomic, so the timer
 138			 * must be disabled at this point.
 139			 */
 140			writeq_relaxed(val, timer->base + CNTP_CVAL_LO);
 141			break;
 142		default:
 143			BUILD_BUG();
 144		}
 145	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 146		struct arch_timer *timer = to_arch_timer(clk);
 147		switch (reg) {
 148		case ARCH_TIMER_REG_CTRL:
 149			writel_relaxed((u32)val, timer->base + CNTV_CTL);
 150			break;
 151		case ARCH_TIMER_REG_CVAL:
 152			/* Same restriction as above */
 153			writeq_relaxed(val, timer->base + CNTV_CVAL_LO);
 154			break;
 155		default:
 156			BUILD_BUG();
 157		}
 158	} else {
 159		arch_timer_reg_write_cp15(access, reg, val);
 160	}
 161}
 162
 163static __always_inline
 164u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 165			struct clock_event_device *clk)
 166{
 167	u32 val;
 168
 169	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 170		struct arch_timer *timer = to_arch_timer(clk);
 171		switch (reg) {
 172		case ARCH_TIMER_REG_CTRL:
 173			val = readl_relaxed(timer->base + CNTP_CTL);
 174			break;
 175		default:
 176			BUILD_BUG();
 
 177		}
 178	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 179		struct arch_timer *timer = to_arch_timer(clk);
 180		switch (reg) {
 181		case ARCH_TIMER_REG_CTRL:
 182			val = readl_relaxed(timer->base + CNTV_CTL);
 183			break;
 184		default:
 185			BUILD_BUG();
 
 186		}
 187	} else {
 188		val = arch_timer_reg_read_cp15(access, reg);
 189	}
 190
 191	return val;
 192}
 193
 194static notrace u64 arch_counter_get_cntpct_stable(void)
 195{
 196	return __arch_counter_get_cntpct_stable();
 197}
 198
 199static notrace u64 arch_counter_get_cntpct(void)
 200{
 201	return __arch_counter_get_cntpct();
 202}
 203
 204static notrace u64 arch_counter_get_cntvct_stable(void)
 205{
 206	return __arch_counter_get_cntvct_stable();
 207}
 208
 209static notrace u64 arch_counter_get_cntvct(void)
 210{
 211	return __arch_counter_get_cntvct();
 212}
 213
 214/*
 215 * Default to cp15 based access because arm64 uses this function for
 216 * sched_clock() before DT is probed and the cp15 method is guaranteed
 217 * to exist on arm64. arm doesn't use this before DT is probed so even
 218 * if we don't have the cp15 accessors we won't have a problem.
 219 */
 220u64 (*arch_timer_read_counter)(void) __ro_after_init = arch_counter_get_cntvct;
 221EXPORT_SYMBOL_GPL(arch_timer_read_counter);
 222
 223static u64 arch_counter_read(struct clocksource *cs)
 224{
 225	return arch_timer_read_counter();
 226}
 227
 228static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 229{
 230	return arch_timer_read_counter();
 231}
 232
 233static struct clocksource clocksource_counter = {
 234	.name	= "arch_sys_counter",
 235	.id	= CSID_ARM_ARCH_COUNTER,
 236	.rating	= 400,
 237	.read	= arch_counter_read,
 
 238	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
 239};
 240
 241static struct cyclecounter cyclecounter __ro_after_init = {
 242	.read	= arch_counter_read_cc,
 
 243};
 244
 245struct ate_acpi_oem_info {
 246	char oem_id[ACPI_OEM_ID_SIZE + 1];
 247	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
 248	u32 oem_revision;
 249};
 250
 251#ifdef CONFIG_FSL_ERRATUM_A008585
 252/*
 253 * The number of retries is an arbitrary value well beyond the highest number
 254 * of iterations the loop has been observed to take.
 255 */
 256#define __fsl_a008585_read_reg(reg) ({			\
 257	u64 _old, _new;					\
 258	int _retries = 200;				\
 259							\
 260	do {						\
 261		_old = read_sysreg(reg);		\
 262		_new = read_sysreg(reg);		\
 263		_retries--;				\
 264	} while (unlikely(_old != _new) && _retries);	\
 265							\
 266	WARN_ON_ONCE(!_retries);			\
 267	_new;						\
 268})
 269
 
 
 
 
 
 
 
 
 
 
 270static u64 notrace fsl_a008585_read_cntpct_el0(void)
 271{
 272	return __fsl_a008585_read_reg(cntpct_el0);
 273}
 274
 275static u64 notrace fsl_a008585_read_cntvct_el0(void)
 276{
 277	return __fsl_a008585_read_reg(cntvct_el0);
 278}
 279#endif
 280
 281#ifdef CONFIG_HISILICON_ERRATUM_161010101
 282/*
 283 * Verify whether the value of the second read is larger than the first by
 284 * less than 32 is the only way to confirm the value is correct, so clear the
 285 * lower 5 bits to check whether the difference is greater than 32 or not.
 286 * Theoretically the erratum should not occur more than twice in succession
 287 * when reading the system counter, but it is possible that some interrupts
 288 * may lead to more than twice read errors, triggering the warning, so setting
 289 * the number of retries far beyond the number of iterations the loop has been
 290 * observed to take.
 291 */
 292#define __hisi_161010101_read_reg(reg) ({				\
 293	u64 _old, _new;						\
 294	int _retries = 50;					\
 295								\
 296	do {							\
 297		_old = read_sysreg(reg);			\
 298		_new = read_sysreg(reg);			\
 299		_retries--;					\
 300	} while (unlikely((_new - _old) >> 5) && _retries);	\
 301								\
 302	WARN_ON_ONCE(!_retries);				\
 303	_new;							\
 304})
 305
 
 
 
 
 
 
 
 
 
 
 306static u64 notrace hisi_161010101_read_cntpct_el0(void)
 307{
 308	return __hisi_161010101_read_reg(cntpct_el0);
 309}
 310
 311static u64 notrace hisi_161010101_read_cntvct_el0(void)
 312{
 313	return __hisi_161010101_read_reg(cntvct_el0);
 314}
 315
 316static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
 317	/*
 318	 * Note that trailing spaces are required to properly match
 319	 * the OEM table information.
 320	 */
 321	{
 322		.oem_id		= "HISI  ",
 323		.oem_table_id	= "HIP05   ",
 324		.oem_revision	= 0,
 325	},
 326	{
 327		.oem_id		= "HISI  ",
 328		.oem_table_id	= "HIP06   ",
 329		.oem_revision	= 0,
 330	},
 331	{
 332		.oem_id		= "HISI  ",
 333		.oem_table_id	= "HIP07   ",
 334		.oem_revision	= 0,
 335	},
 336	{ /* Sentinel indicating the end of the OEM array */ },
 337};
 338#endif
 339
 340#ifdef CONFIG_ARM64_ERRATUM_858921
 341static u64 notrace arm64_858921_read_cntpct_el0(void)
 342{
 343	u64 old, new;
 344
 345	old = read_sysreg(cntpct_el0);
 346	new = read_sysreg(cntpct_el0);
 347	return (((old ^ new) >> 32) & 1) ? old : new;
 348}
 349
 350static u64 notrace arm64_858921_read_cntvct_el0(void)
 351{
 352	u64 old, new;
 353
 354	old = read_sysreg(cntvct_el0);
 355	new = read_sysreg(cntvct_el0);
 356	return (((old ^ new) >> 32) & 1) ? old : new;
 357}
 358#endif
 359
 360#ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 361/*
 362 * The low bits of the counter registers are indeterminate while bit 10 or
 363 * greater is rolling over. Since the counter value can jump both backward
 364 * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
 365 * with all ones or all zeros in the low bits. Bound the loop by the maximum
 366 * number of CPU cycles in 3 consecutive 24 MHz counter periods.
 367 */
 368#define __sun50i_a64_read_reg(reg) ({					\
 369	u64 _val;							\
 370	int _retries = 150;						\
 371									\
 372	do {								\
 373		_val = read_sysreg(reg);				\
 374		_retries--;						\
 375	} while (((_val + 1) & GENMASK(8, 0)) <= 1 && _retries);	\
 376									\
 377	WARN_ON_ONCE(!_retries);					\
 378	_val;								\
 379})
 380
 381static u64 notrace sun50i_a64_read_cntpct_el0(void)
 382{
 383	return __sun50i_a64_read_reg(cntpct_el0);
 384}
 385
 386static u64 notrace sun50i_a64_read_cntvct_el0(void)
 387{
 388	return __sun50i_a64_read_reg(cntvct_el0);
 389}
 
 
 
 
 
 
 
 
 
 
 390#endif
 391
 392#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
 393DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
 394EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
 395
 396static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
 397
 398/*
 399 * Force the inlining of this function so that the register accesses
 400 * can be themselves correctly inlined.
 401 */
 402static __always_inline
 403void erratum_set_next_event_generic(const int access, unsigned long evt,
 404				    struct clock_event_device *clk)
 405{
 406	unsigned long ctrl;
 407	u64 cval;
 408
 409	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 410	ctrl |= ARCH_TIMER_CTRL_ENABLE;
 411	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 412
 413	if (access == ARCH_TIMER_PHYS_ACCESS) {
 414		cval = evt + arch_counter_get_cntpct_stable();
 415		write_sysreg(cval, cntp_cval_el0);
 416	} else {
 417		cval = evt + arch_counter_get_cntvct_stable();
 418		write_sysreg(cval, cntv_cval_el0);
 419	}
 420
 421	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 422}
 423
 424static __maybe_unused int erratum_set_next_event_virt(unsigned long evt,
 425					    struct clock_event_device *clk)
 426{
 427	erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 428	return 0;
 429}
 430
 431static __maybe_unused int erratum_set_next_event_phys(unsigned long evt,
 432					    struct clock_event_device *clk)
 433{
 434	erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 435	return 0;
 436}
 437
 438static const struct arch_timer_erratum_workaround ool_workarounds[] = {
 439#ifdef CONFIG_FSL_ERRATUM_A008585
 440	{
 441		.match_type = ate_match_dt,
 442		.id = "fsl,erratum-a008585",
 443		.desc = "Freescale erratum a005858",
 
 
 444		.read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
 445		.read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
 446		.set_next_event_phys = erratum_set_next_event_phys,
 447		.set_next_event_virt = erratum_set_next_event_virt,
 448	},
 449#endif
 450#ifdef CONFIG_HISILICON_ERRATUM_161010101
 451	{
 452		.match_type = ate_match_dt,
 453		.id = "hisilicon,erratum-161010101",
 454		.desc = "HiSilicon erratum 161010101",
 
 
 455		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 456		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 457		.set_next_event_phys = erratum_set_next_event_phys,
 458		.set_next_event_virt = erratum_set_next_event_virt,
 459	},
 460	{
 461		.match_type = ate_match_acpi_oem_info,
 462		.id = hisi_161010101_oem_info,
 463		.desc = "HiSilicon erratum 161010101",
 
 
 464		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 465		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 466		.set_next_event_phys = erratum_set_next_event_phys,
 467		.set_next_event_virt = erratum_set_next_event_virt,
 468	},
 469#endif
 470#ifdef CONFIG_ARM64_ERRATUM_858921
 471	{
 472		.match_type = ate_match_local_cap_id,
 473		.id = (void *)ARM64_WORKAROUND_858921,
 474		.desc = "ARM erratum 858921",
 475		.read_cntpct_el0 = arm64_858921_read_cntpct_el0,
 476		.read_cntvct_el0 = arm64_858921_read_cntvct_el0,
 477		.set_next_event_phys = erratum_set_next_event_phys,
 478		.set_next_event_virt = erratum_set_next_event_virt,
 479	},
 480#endif
 481#ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 482	{
 483		.match_type = ate_match_dt,
 484		.id = "allwinner,erratum-unknown1",
 485		.desc = "Allwinner erratum UNKNOWN1",
 
 
 486		.read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
 487		.read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
 488		.set_next_event_phys = erratum_set_next_event_phys,
 489		.set_next_event_virt = erratum_set_next_event_virt,
 490	},
 491#endif
 492#ifdef CONFIG_ARM64_ERRATUM_1418040
 493	{
 494		.match_type = ate_match_local_cap_id,
 495		.id = (void *)ARM64_WORKAROUND_1418040,
 496		.desc = "ARM erratum 1418040",
 497		.disable_compat_vdso = true,
 498	},
 499#endif
 500};
 501
 502typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
 503			       const void *);
 504
 505static
 506bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
 507				 const void *arg)
 508{
 509	const struct device_node *np = arg;
 510
 511	return of_property_read_bool(np, wa->id);
 512}
 513
 514static
 515bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
 516					const void *arg)
 517{
 518	return this_cpu_has_cap((uintptr_t)wa->id);
 519}
 520
 521
 522static
 523bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
 524				       const void *arg)
 525{
 526	static const struct ate_acpi_oem_info empty_oem_info = {};
 527	const struct ate_acpi_oem_info *info = wa->id;
 528	const struct acpi_table_header *table = arg;
 529
 530	/* Iterate over the ACPI OEM info array, looking for a match */
 531	while (memcmp(info, &empty_oem_info, sizeof(*info))) {
 532		if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
 533		    !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
 534		    info->oem_revision == table->oem_revision)
 535			return true;
 536
 537		info++;
 538	}
 539
 540	return false;
 541}
 542
 543static const struct arch_timer_erratum_workaround *
 544arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
 545			  ate_match_fn_t match_fn,
 546			  void *arg)
 547{
 548	int i;
 549
 550	for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
 551		if (ool_workarounds[i].match_type != type)
 552			continue;
 553
 554		if (match_fn(&ool_workarounds[i], arg))
 555			return &ool_workarounds[i];
 556	}
 557
 558	return NULL;
 559}
 560
 561static
 562void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
 563				  bool local)
 564{
 565	int i;
 566
 567	if (local) {
 568		__this_cpu_write(timer_unstable_counter_workaround, wa);
 569	} else {
 570		for_each_possible_cpu(i)
 571			per_cpu(timer_unstable_counter_workaround, i) = wa;
 572	}
 573
 574	if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
 575		atomic_set(&timer_unstable_counter_workaround_in_use, 1);
 576
 577	/*
 578	 * Don't use the vdso fastpath if errata require using the
 579	 * out-of-line counter accessor. We may change our mind pretty
 580	 * late in the game (with a per-CPU erratum, for example), so
 581	 * change both the default value and the vdso itself.
 582	 */
 583	if (wa->read_cntvct_el0) {
 584		clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
 585		vdso_default = VDSO_CLOCKMODE_NONE;
 586	} else if (wa->disable_compat_vdso && vdso_default != VDSO_CLOCKMODE_NONE) {
 587		vdso_default = VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT;
 588		clocksource_counter.vdso_clock_mode = vdso_default;
 589	}
 590}
 591
 592static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
 593					    void *arg)
 594{
 595	const struct arch_timer_erratum_workaround *wa, *__wa;
 596	ate_match_fn_t match_fn = NULL;
 597	bool local = false;
 598
 599	switch (type) {
 600	case ate_match_dt:
 601		match_fn = arch_timer_check_dt_erratum;
 602		break;
 603	case ate_match_local_cap_id:
 604		match_fn = arch_timer_check_local_cap_erratum;
 605		local = true;
 606		break;
 607	case ate_match_acpi_oem_info:
 608		match_fn = arch_timer_check_acpi_oem_erratum;
 609		break;
 610	default:
 611		WARN_ON(1);
 612		return;
 613	}
 614
 615	wa = arch_timer_iterate_errata(type, match_fn, arg);
 616	if (!wa)
 617		return;
 618
 619	__wa = __this_cpu_read(timer_unstable_counter_workaround);
 620	if (__wa && wa != __wa)
 621		pr_warn("Can't enable workaround for %s (clashes with %s\n)",
 622			wa->desc, __wa->desc);
 623
 624	if (__wa)
 625		return;
 626
 627	arch_timer_enable_workaround(wa, local);
 628	pr_info("Enabling %s workaround for %s\n",
 629		local ? "local" : "global", wa->desc);
 630}
 631
 632static bool arch_timer_this_cpu_has_cntvct_wa(void)
 633{
 634	return has_erratum_handler(read_cntvct_el0);
 635}
 636
 637static bool arch_timer_counter_has_wa(void)
 638{
 639	return atomic_read(&timer_unstable_counter_workaround_in_use);
 640}
 641#else
 642#define arch_timer_check_ool_workaround(t,a)		do { } while(0)
 643#define arch_timer_this_cpu_has_cntvct_wa()		({false;})
 644#define arch_timer_counter_has_wa()			({false;})
 645#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
 646
 647static __always_inline irqreturn_t timer_handler(const int access,
 648					struct clock_event_device *evt)
 649{
 650	unsigned long ctrl;
 651
 652	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 653	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 654		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 655		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 656		evt->event_handler(evt);
 657		return IRQ_HANDLED;
 658	}
 659
 660	return IRQ_NONE;
 661}
 662
 663static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 664{
 665	struct clock_event_device *evt = dev_id;
 666
 667	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 668}
 669
 670static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 671{
 672	struct clock_event_device *evt = dev_id;
 673
 674	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 675}
 676
 677static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 678{
 679	struct clock_event_device *evt = dev_id;
 680
 681	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 682}
 683
 684static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 685{
 686	struct clock_event_device *evt = dev_id;
 687
 688	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 689}
 690
 691static __always_inline int arch_timer_shutdown(const int access,
 692					       struct clock_event_device *clk)
 693{
 694	unsigned long ctrl;
 695
 696	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 697	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 698	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 699
 700	return 0;
 701}
 702
 703static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 704{
 705	return arch_timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 706}
 707
 708static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 709{
 710	return arch_timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 711}
 712
 713static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 714{
 715	return arch_timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 716}
 717
 718static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 719{
 720	return arch_timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 721}
 722
 723static __always_inline void set_next_event(const int access, unsigned long evt,
 724					   struct clock_event_device *clk)
 725{
 726	unsigned long ctrl;
 727	u64 cnt;
 728
 729	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 730	ctrl |= ARCH_TIMER_CTRL_ENABLE;
 731	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 732
 733	if (access == ARCH_TIMER_PHYS_ACCESS)
 734		cnt = __arch_counter_get_cntpct();
 735	else
 736		cnt = __arch_counter_get_cntvct();
 737
 738	arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
 739	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 740}
 741
 742static int arch_timer_set_next_event_virt(unsigned long evt,
 743					  struct clock_event_device *clk)
 744{
 745	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 746	return 0;
 747}
 748
 749static int arch_timer_set_next_event_phys(unsigned long evt,
 750					  struct clock_event_device *clk)
 751{
 752	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 753	return 0;
 754}
 755
 756static u64 arch_counter_get_cnt_mem(struct arch_timer *t, int offset_lo)
 757{
 758	u32 cnt_lo, cnt_hi, tmp_hi;
 759
 760	do {
 761		cnt_hi = readl_relaxed(t->base + offset_lo + 4);
 762		cnt_lo = readl_relaxed(t->base + offset_lo);
 763		tmp_hi = readl_relaxed(t->base + offset_lo + 4);
 764	} while (cnt_hi != tmp_hi);
 765
 766	return ((u64) cnt_hi << 32) | cnt_lo;
 767}
 768
 769static __always_inline void set_next_event_mem(const int access, unsigned long evt,
 770					   struct clock_event_device *clk)
 771{
 772	struct arch_timer *timer = to_arch_timer(clk);
 773	unsigned long ctrl;
 774	u64 cnt;
 775
 776	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 777	ctrl |= ARCH_TIMER_CTRL_ENABLE;
 778	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 779
 780	if (access ==  ARCH_TIMER_MEM_VIRT_ACCESS)
 781		cnt = arch_counter_get_cnt_mem(timer, CNTVCT_LO);
 782	else
 783		cnt = arch_counter_get_cnt_mem(timer, CNTPCT_LO);
 784
 785	arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
 786	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 787}
 788
 789static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 790					      struct clock_event_device *clk)
 791{
 792	set_next_event_mem(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 793	return 0;
 794}
 795
 796static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 797					      struct clock_event_device *clk)
 798{
 799	set_next_event_mem(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 800	return 0;
 801}
 802
 803static u64 __arch_timer_check_delta(void)
 804{
 805#ifdef CONFIG_ARM64
 806	const struct midr_range broken_cval_midrs[] = {
 807		/*
 808		 * XGene-1 implements CVAL in terms of TVAL, meaning
 809		 * that the maximum timer range is 32bit. Shame on them.
 810		 *
 811		 * Note that TVAL is signed, thus has only 31 of its
 812		 * 32 bits to express magnitude.
 813		 */
 814		MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APM,
 815						 APM_CPU_PART_POTENZA)),
 816		{},
 817	};
 818
 819	if (is_midr_in_range_list(read_cpuid_id(), broken_cval_midrs)) {
 820		pr_warn_once("Broken CNTx_CVAL_EL1, using 31 bit TVAL instead.\n");
 821		return CLOCKSOURCE_MASK(31);
 822	}
 823#endif
 824	return CLOCKSOURCE_MASK(arch_counter_get_width());
 825}
 826
 827static void __arch_timer_setup(unsigned type,
 828			       struct clock_event_device *clk)
 829{
 830	u64 max_delta;
 831
 832	clk->features = CLOCK_EVT_FEAT_ONESHOT;
 833
 834	if (type == ARCH_TIMER_TYPE_CP15) {
 835		typeof(clk->set_next_event) sne;
 836
 837		arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
 838
 839		if (arch_timer_c3stop)
 840			clk->features |= CLOCK_EVT_FEAT_C3STOP;
 841		clk->name = "arch_sys_timer";
 842		clk->rating = 450;
 843		clk->cpumask = cpumask_of(smp_processor_id());
 844		clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 845		switch (arch_timer_uses_ppi) {
 846		case ARCH_TIMER_VIRT_PPI:
 847			clk->set_state_shutdown = arch_timer_shutdown_virt;
 848			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 849			sne = erratum_handler(set_next_event_virt);
 850			break;
 851		case ARCH_TIMER_PHYS_SECURE_PPI:
 852		case ARCH_TIMER_PHYS_NONSECURE_PPI:
 853		case ARCH_TIMER_HYP_PPI:
 854			clk->set_state_shutdown = arch_timer_shutdown_phys;
 855			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 856			sne = erratum_handler(set_next_event_phys);
 857			break;
 858		default:
 859			BUG();
 860		}
 861
 862		clk->set_next_event = sne;
 863		max_delta = __arch_timer_check_delta();
 864	} else {
 865		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 866		clk->name = "arch_mem_timer";
 867		clk->rating = 400;
 868		clk->cpumask = cpu_possible_mask;
 869		if (arch_timer_mem_use_virtual) {
 870			clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 871			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 872			clk->set_next_event =
 873				arch_timer_set_next_event_virt_mem;
 874		} else {
 875			clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 876			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 877			clk->set_next_event =
 878				arch_timer_set_next_event_phys_mem;
 879		}
 880
 881		max_delta = CLOCKSOURCE_MASK(56);
 882	}
 883
 884	clk->set_state_shutdown(clk);
 885
 886	clockevents_config_and_register(clk, arch_timer_rate, 0xf, max_delta);
 887}
 888
 889static void arch_timer_evtstrm_enable(unsigned int divider)
 890{
 891	u32 cntkctl = arch_timer_get_cntkctl();
 892
 893#ifdef CONFIG_ARM64
 894	/* ECV is likely to require a large divider. Use the EVNTIS flag. */
 895	if (cpus_have_const_cap(ARM64_HAS_ECV) && divider > 15) {
 896		cntkctl |= ARCH_TIMER_EVT_INTERVAL_SCALE;
 897		divider -= 8;
 898	}
 899#endif
 900
 901	divider = min(divider, 15U);
 902	cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 903	/* Set the divider and enable virtual event stream */
 904	cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 905			| ARCH_TIMER_VIRT_EVT_EN;
 906	arch_timer_set_cntkctl(cntkctl);
 907	arch_timer_set_evtstrm_feature();
 908	cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
 909}
 910
 911static void arch_timer_configure_evtstream(void)
 912{
 913	int evt_stream_div, lsb;
 914
 915	/*
 916	 * As the event stream can at most be generated at half the frequency
 917	 * of the counter, use half the frequency when computing the divider.
 918	 */
 919	evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ / 2;
 920
 921	/*
 922	 * Find the closest power of two to the divisor. If the adjacent bit
 923	 * of lsb (last set bit, starts from 0) is set, then we use (lsb + 1).
 924	 */
 925	lsb = fls(evt_stream_div) - 1;
 926	if (lsb > 0 && (evt_stream_div & BIT(lsb - 1)))
 927		lsb++;
 928
 929	/* enable event stream */
 930	arch_timer_evtstrm_enable(max(0, lsb));
 931}
 932
 933static void arch_counter_set_user_access(void)
 934{
 935	u32 cntkctl = arch_timer_get_cntkctl();
 936
 937	/* Disable user access to the timers and both counters */
 938	/* Also disable virtual event stream */
 939	cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 940			| ARCH_TIMER_USR_VT_ACCESS_EN
 941		        | ARCH_TIMER_USR_VCT_ACCESS_EN
 942			| ARCH_TIMER_VIRT_EVT_EN
 943			| ARCH_TIMER_USR_PCT_ACCESS_EN);
 944
 945	/*
 946	 * Enable user access to the virtual counter if it doesn't
 947	 * need to be workaround. The vdso may have been already
 948	 * disabled though.
 949	 */
 950	if (arch_timer_this_cpu_has_cntvct_wa())
 951		pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
 952	else
 953		cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 954
 955	arch_timer_set_cntkctl(cntkctl);
 956}
 957
 958static bool arch_timer_has_nonsecure_ppi(void)
 959{
 960	return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
 961		arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 962}
 963
 964static u32 check_ppi_trigger(int irq)
 965{
 966	u32 flags = irq_get_trigger_type(irq);
 967
 968	if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 969		pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 970		pr_warn("WARNING: Please fix your firmware\n");
 971		flags = IRQF_TRIGGER_LOW;
 972	}
 973
 974	return flags;
 975}
 976
 977static int arch_timer_starting_cpu(unsigned int cpu)
 978{
 979	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 980	u32 flags;
 981
 982	__arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
 983
 984	flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 985	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 986
 987	if (arch_timer_has_nonsecure_ppi()) {
 988		flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 989		enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
 990				  flags);
 991	}
 992
 993	arch_counter_set_user_access();
 994	if (evtstrm_enable)
 995		arch_timer_configure_evtstream();
 996
 997	return 0;
 998}
 999
1000static int validate_timer_rate(void)
1001{
1002	if (!arch_timer_rate)
1003		return -EINVAL;
1004
1005	/* Arch timer frequency < 1MHz can cause trouble */
1006	WARN_ON(arch_timer_rate < 1000000);
1007
1008	return 0;
1009}
1010
1011/*
1012 * For historical reasons, when probing with DT we use whichever (non-zero)
1013 * rate was probed first, and don't verify that others match. If the first node
1014 * probed has a clock-frequency property, this overrides the HW register.
1015 */
1016static void __init arch_timer_of_configure_rate(u32 rate, struct device_node *np)
1017{
1018	/* Who has more than one independent system counter? */
1019	if (arch_timer_rate)
1020		return;
1021
1022	if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
1023		arch_timer_rate = rate;
1024
1025	/* Check the timer frequency. */
1026	if (validate_timer_rate())
1027		pr_warn("frequency not available\n");
1028}
1029
1030static void __init arch_timer_banner(unsigned type)
1031{
1032	pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
1033		type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
1034		type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
1035			" and " : "",
1036		type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
1037		(unsigned long)arch_timer_rate / 1000000,
1038		(unsigned long)(arch_timer_rate / 10000) % 100,
1039		type & ARCH_TIMER_TYPE_CP15 ?
1040			(arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
1041			"",
1042		type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
1043		type & ARCH_TIMER_TYPE_MEM ?
1044			arch_timer_mem_use_virtual ? "virt" : "phys" :
1045			"");
1046}
1047
1048u32 arch_timer_get_rate(void)
1049{
1050	return arch_timer_rate;
1051}
1052
1053bool arch_timer_evtstrm_available(void)
1054{
1055	/*
1056	 * We might get called from a preemptible context. This is fine
1057	 * because availability of the event stream should be always the same
1058	 * for a preemptible context and context where we might resume a task.
1059	 */
1060	return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
1061}
1062
1063static u64 arch_counter_get_cntvct_mem(void)
1064{
1065	return arch_counter_get_cnt_mem(arch_timer_mem, CNTVCT_LO);
 
 
 
 
 
 
 
 
1066}
1067
1068static struct arch_timer_kvm_info arch_timer_kvm_info;
1069
1070struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
1071{
1072	return &arch_timer_kvm_info;
1073}
1074
1075static void __init arch_counter_register(unsigned type)
1076{
1077	u64 start_count;
1078	int width;
1079
1080	/* Register the CP15 based counter if we have one */
1081	if (type & ARCH_TIMER_TYPE_CP15) {
1082		u64 (*rd)(void);
1083
1084		if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
1085		    arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
1086			if (arch_timer_counter_has_wa())
1087				rd = arch_counter_get_cntvct_stable;
1088			else
1089				rd = arch_counter_get_cntvct;
1090		} else {
1091			if (arch_timer_counter_has_wa())
1092				rd = arch_counter_get_cntpct_stable;
1093			else
1094				rd = arch_counter_get_cntpct;
1095		}
1096
1097		arch_timer_read_counter = rd;
1098		clocksource_counter.vdso_clock_mode = vdso_default;
1099	} else {
1100		arch_timer_read_counter = arch_counter_get_cntvct_mem;
1101	}
1102
1103	width = arch_counter_get_width();
1104	clocksource_counter.mask = CLOCKSOURCE_MASK(width);
1105	cyclecounter.mask = CLOCKSOURCE_MASK(width);
1106
1107	if (!arch_counter_suspend_stop)
1108		clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
1109	start_count = arch_timer_read_counter();
1110	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
1111	cyclecounter.mult = clocksource_counter.mult;
1112	cyclecounter.shift = clocksource_counter.shift;
1113	timecounter_init(&arch_timer_kvm_info.timecounter,
1114			 &cyclecounter, start_count);
1115
1116	sched_clock_register(arch_timer_read_counter, width, arch_timer_rate);
 
1117}
1118
1119static void arch_timer_stop(struct clock_event_device *clk)
1120{
1121	pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1122
1123	disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
1124	if (arch_timer_has_nonsecure_ppi())
1125		disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1126
1127	clk->set_state_shutdown(clk);
1128}
1129
1130static int arch_timer_dying_cpu(unsigned int cpu)
1131{
1132	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1133
1134	cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1135
1136	arch_timer_stop(clk);
1137	return 0;
1138}
1139
1140#ifdef CONFIG_CPU_PM
1141static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
1142static int arch_timer_cpu_pm_notify(struct notifier_block *self,
1143				    unsigned long action, void *hcpu)
1144{
1145	if (action == CPU_PM_ENTER) {
1146		__this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1147
1148		cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1149	} else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1150		arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1151
1152		if (arch_timer_have_evtstrm_feature())
1153			cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
1154	}
1155	return NOTIFY_OK;
1156}
1157
1158static struct notifier_block arch_timer_cpu_pm_notifier = {
1159	.notifier_call = arch_timer_cpu_pm_notify,
1160};
1161
1162static int __init arch_timer_cpu_pm_init(void)
1163{
1164	return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
1165}
1166
1167static void __init arch_timer_cpu_pm_deinit(void)
1168{
1169	WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
1170}
1171
1172#else
1173static int __init arch_timer_cpu_pm_init(void)
1174{
1175	return 0;
1176}
1177
1178static void __init arch_timer_cpu_pm_deinit(void)
1179{
1180}
1181#endif
1182
1183static int __init arch_timer_register(void)
1184{
1185	int err;
1186	int ppi;
1187
1188	arch_timer_evt = alloc_percpu(struct clock_event_device);
1189	if (!arch_timer_evt) {
1190		err = -ENOMEM;
1191		goto out;
1192	}
1193
1194	ppi = arch_timer_ppi[arch_timer_uses_ppi];
1195	switch (arch_timer_uses_ppi) {
1196	case ARCH_TIMER_VIRT_PPI:
1197		err = request_percpu_irq(ppi, arch_timer_handler_virt,
1198					 "arch_timer", arch_timer_evt);
1199		break;
1200	case ARCH_TIMER_PHYS_SECURE_PPI:
1201	case ARCH_TIMER_PHYS_NONSECURE_PPI:
1202		err = request_percpu_irq(ppi, arch_timer_handler_phys,
1203					 "arch_timer", arch_timer_evt);
1204		if (!err && arch_timer_has_nonsecure_ppi()) {
1205			ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1206			err = request_percpu_irq(ppi, arch_timer_handler_phys,
1207						 "arch_timer", arch_timer_evt);
1208			if (err)
1209				free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1210						arch_timer_evt);
1211		}
1212		break;
1213	case ARCH_TIMER_HYP_PPI:
1214		err = request_percpu_irq(ppi, arch_timer_handler_phys,
1215					 "arch_timer", arch_timer_evt);
1216		break;
1217	default:
1218		BUG();
1219	}
1220
1221	if (err) {
1222		pr_err("can't register interrupt %d (%d)\n", ppi, err);
1223		goto out_free;
1224	}
1225
1226	err = arch_timer_cpu_pm_init();
1227	if (err)
1228		goto out_unreg_notify;
1229
1230	/* Register and immediately configure the timer on the boot CPU */
1231	err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1232				"clockevents/arm/arch_timer:starting",
1233				arch_timer_starting_cpu, arch_timer_dying_cpu);
1234	if (err)
1235		goto out_unreg_cpupm;
1236	return 0;
1237
1238out_unreg_cpupm:
1239	arch_timer_cpu_pm_deinit();
1240
1241out_unreg_notify:
1242	free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1243	if (arch_timer_has_nonsecure_ppi())
1244		free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1245				arch_timer_evt);
1246
1247out_free:
1248	free_percpu(arch_timer_evt);
1249out:
1250	return err;
1251}
1252
1253static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1254{
1255	int ret;
1256	irq_handler_t func;
 
1257
1258	arch_timer_mem = kzalloc(sizeof(*arch_timer_mem), GFP_KERNEL);
1259	if (!arch_timer_mem)
1260		return -ENOMEM;
1261
1262	arch_timer_mem->base = base;
1263	arch_timer_mem->evt.irq = irq;
1264	__arch_timer_setup(ARCH_TIMER_TYPE_MEM, &arch_timer_mem->evt);
1265
1266	if (arch_timer_mem_use_virtual)
1267		func = arch_timer_handler_virt_mem;
1268	else
1269		func = arch_timer_handler_phys_mem;
1270
1271	ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &arch_timer_mem->evt);
1272	if (ret) {
1273		pr_err("Failed to request mem timer irq\n");
1274		kfree(arch_timer_mem);
1275		arch_timer_mem = NULL;
1276	}
1277
1278	return ret;
1279}
1280
1281static const struct of_device_id arch_timer_of_match[] __initconst = {
1282	{ .compatible   = "arm,armv7-timer",    },
1283	{ .compatible   = "arm,armv8-timer",    },
1284	{},
1285};
1286
1287static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1288	{ .compatible   = "arm,armv7-timer-mem", },
1289	{},
1290};
1291
1292static bool __init arch_timer_needs_of_probing(void)
1293{
1294	struct device_node *dn;
1295	bool needs_probing = false;
1296	unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1297
1298	/* We have two timers, and both device-tree nodes are probed. */
1299	if ((arch_timers_present & mask) == mask)
1300		return false;
1301
1302	/*
1303	 * Only one type of timer is probed,
1304	 * check if we have another type of timer node in device-tree.
1305	 */
1306	if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1307		dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1308	else
1309		dn = of_find_matching_node(NULL, arch_timer_of_match);
1310
1311	if (dn && of_device_is_available(dn))
1312		needs_probing = true;
1313
1314	of_node_put(dn);
1315
1316	return needs_probing;
1317}
1318
1319static int __init arch_timer_common_init(void)
1320{
1321	arch_timer_banner(arch_timers_present);
1322	arch_counter_register(arch_timers_present);
1323	return arch_timer_arch_init();
1324}
1325
1326/**
1327 * arch_timer_select_ppi() - Select suitable PPI for the current system.
1328 *
1329 * If HYP mode is available, we know that the physical timer
1330 * has been configured to be accessible from PL1. Use it, so
1331 * that a guest can use the virtual timer instead.
1332 *
1333 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1334 * accesses to CNTP_*_EL1 registers are silently redirected to
1335 * their CNTHP_*_EL2 counterparts, and use a different PPI
1336 * number.
1337 *
1338 * If no interrupt provided for virtual timer, we'll have to
1339 * stick to the physical timer. It'd better be accessible...
1340 * For arm64 we never use the secure interrupt.
1341 *
1342 * Return: a suitable PPI type for the current system.
1343 */
1344static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1345{
1346	if (is_kernel_in_hyp_mode())
1347		return ARCH_TIMER_HYP_PPI;
1348
1349	if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1350		return ARCH_TIMER_VIRT_PPI;
1351
1352	if (IS_ENABLED(CONFIG_ARM64))
1353		return ARCH_TIMER_PHYS_NONSECURE_PPI;
1354
1355	return ARCH_TIMER_PHYS_SECURE_PPI;
1356}
1357
1358static void __init arch_timer_populate_kvm_info(void)
1359{
1360	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1361	if (is_kernel_in_hyp_mode())
1362		arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1363}
1364
1365static int __init arch_timer_of_init(struct device_node *np)
1366{
1367	int i, irq, ret;
1368	u32 rate;
1369	bool has_names;
1370
1371	if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1372		pr_warn("multiple nodes in dt, skipping\n");
1373		return 0;
1374	}
1375
1376	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1377
1378	has_names = of_property_read_bool(np, "interrupt-names");
1379
1380	for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) {
1381		if (has_names)
1382			irq = of_irq_get_byname(np, arch_timer_ppi_names[i]);
1383		else
1384			irq = of_irq_get(np, i);
1385		if (irq > 0)
1386			arch_timer_ppi[i] = irq;
1387	}
1388
1389	arch_timer_populate_kvm_info();
1390
1391	rate = arch_timer_get_cntfrq();
1392	arch_timer_of_configure_rate(rate, np);
1393
1394	arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1395
1396	/* Check for globally applicable workarounds */
1397	arch_timer_check_ool_workaround(ate_match_dt, np);
1398
1399	/*
1400	 * If we cannot rely on firmware initializing the timer registers then
1401	 * we should use the physical timers instead.
1402	 */
1403	if (IS_ENABLED(CONFIG_ARM) &&
1404	    of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1405		arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1406	else
1407		arch_timer_uses_ppi = arch_timer_select_ppi();
1408
1409	if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1410		pr_err("No interrupt available, giving up\n");
1411		return -EINVAL;
1412	}
1413
1414	/* On some systems, the counter stops ticking when in suspend. */
1415	arch_counter_suspend_stop = of_property_read_bool(np,
1416							 "arm,no-tick-in-suspend");
1417
1418	ret = arch_timer_register();
1419	if (ret)
1420		return ret;
1421
1422	if (arch_timer_needs_of_probing())
1423		return 0;
1424
1425	return arch_timer_common_init();
1426}
1427TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1428TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1429
1430static u32 __init
1431arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1432{
1433	void __iomem *base;
1434	u32 rate;
1435
1436	base = ioremap(frame->cntbase, frame->size);
1437	if (!base) {
1438		pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1439		return 0;
1440	}
1441
1442	rate = readl_relaxed(base + CNTFRQ);
1443
1444	iounmap(base);
1445
1446	return rate;
1447}
1448
1449static struct arch_timer_mem_frame * __init
1450arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1451{
1452	struct arch_timer_mem_frame *frame, *best_frame = NULL;
1453	void __iomem *cntctlbase;
1454	u32 cnttidr;
1455	int i;
1456
1457	cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1458	if (!cntctlbase) {
1459		pr_err("Can't map CNTCTLBase @ %pa\n",
1460			&timer_mem->cntctlbase);
1461		return NULL;
1462	}
1463
1464	cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1465
1466	/*
1467	 * Try to find a virtual capable frame. Otherwise fall back to a
1468	 * physical capable frame.
1469	 */
1470	for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1471		u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1472			     CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1473
1474		frame = &timer_mem->frame[i];
1475		if (!frame->valid)
1476			continue;
1477
1478		/* Try enabling everything, and see what sticks */
1479		writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1480		cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1481
1482		if ((cnttidr & CNTTIDR_VIRT(i)) &&
1483		    !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1484			best_frame = frame;
1485			arch_timer_mem_use_virtual = true;
1486			break;
1487		}
1488
1489		if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1490			continue;
1491
1492		best_frame = frame;
1493	}
1494
1495	iounmap(cntctlbase);
1496
1497	return best_frame;
1498}
1499
1500static int __init
1501arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1502{
1503	void __iomem *base;
1504	int ret, irq = 0;
1505
1506	if (arch_timer_mem_use_virtual)
1507		irq = frame->virt_irq;
1508	else
1509		irq = frame->phys_irq;
1510
1511	if (!irq) {
1512		pr_err("Frame missing %s irq.\n",
1513		       arch_timer_mem_use_virtual ? "virt" : "phys");
1514		return -EINVAL;
1515	}
1516
1517	if (!request_mem_region(frame->cntbase, frame->size,
1518				"arch_mem_timer"))
1519		return -EBUSY;
1520
1521	base = ioremap(frame->cntbase, frame->size);
1522	if (!base) {
1523		pr_err("Can't map frame's registers\n");
1524		return -ENXIO;
1525	}
1526
1527	ret = arch_timer_mem_register(base, irq);
1528	if (ret) {
1529		iounmap(base);
1530		return ret;
1531	}
1532
 
1533	arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1534
1535	return 0;
1536}
1537
1538static int __init arch_timer_mem_of_init(struct device_node *np)
1539{
1540	struct arch_timer_mem *timer_mem;
1541	struct arch_timer_mem_frame *frame;
1542	struct device_node *frame_node;
1543	struct resource res;
1544	int ret = -EINVAL;
1545	u32 rate;
1546
1547	timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1548	if (!timer_mem)
1549		return -ENOMEM;
1550
1551	if (of_address_to_resource(np, 0, &res))
1552		goto out;
1553	timer_mem->cntctlbase = res.start;
1554	timer_mem->size = resource_size(&res);
1555
1556	for_each_available_child_of_node(np, frame_node) {
1557		u32 n;
1558		struct arch_timer_mem_frame *frame;
1559
1560		if (of_property_read_u32(frame_node, "frame-number", &n)) {
1561			pr_err(FW_BUG "Missing frame-number.\n");
1562			of_node_put(frame_node);
1563			goto out;
1564		}
1565		if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1566			pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1567			       ARCH_TIMER_MEM_MAX_FRAMES - 1);
1568			of_node_put(frame_node);
1569			goto out;
1570		}
1571		frame = &timer_mem->frame[n];
1572
1573		if (frame->valid) {
1574			pr_err(FW_BUG "Duplicated frame-number.\n");
1575			of_node_put(frame_node);
1576			goto out;
1577		}
1578
1579		if (of_address_to_resource(frame_node, 0, &res)) {
1580			of_node_put(frame_node);
1581			goto out;
1582		}
1583		frame->cntbase = res.start;
1584		frame->size = resource_size(&res);
1585
1586		frame->virt_irq = irq_of_parse_and_map(frame_node,
1587						       ARCH_TIMER_VIRT_SPI);
1588		frame->phys_irq = irq_of_parse_and_map(frame_node,
1589						       ARCH_TIMER_PHYS_SPI);
1590
1591		frame->valid = true;
1592	}
1593
1594	frame = arch_timer_mem_find_best_frame(timer_mem);
1595	if (!frame) {
1596		pr_err("Unable to find a suitable frame in timer @ %pa\n",
1597			&timer_mem->cntctlbase);
1598		ret = -EINVAL;
1599		goto out;
1600	}
1601
1602	rate = arch_timer_mem_frame_get_cntfrq(frame);
1603	arch_timer_of_configure_rate(rate, np);
1604
1605	ret = arch_timer_mem_frame_register(frame);
1606	if (!ret && !arch_timer_needs_of_probing())
1607		ret = arch_timer_common_init();
1608out:
1609	kfree(timer_mem);
1610	return ret;
1611}
1612TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1613		       arch_timer_mem_of_init);
1614
1615#ifdef CONFIG_ACPI_GTDT
1616static int __init
1617arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1618{
1619	struct arch_timer_mem_frame *frame;
1620	u32 rate;
1621	int i;
1622
1623	for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1624		frame = &timer_mem->frame[i];
1625
1626		if (!frame->valid)
1627			continue;
1628
1629		rate = arch_timer_mem_frame_get_cntfrq(frame);
1630		if (rate == arch_timer_rate)
1631			continue;
1632
1633		pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1634			&frame->cntbase,
1635			(unsigned long)rate, (unsigned long)arch_timer_rate);
1636
1637		return -EINVAL;
1638	}
1639
1640	return 0;
1641}
1642
1643static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1644{
1645	struct arch_timer_mem *timers, *timer;
1646	struct arch_timer_mem_frame *frame, *best_frame = NULL;
1647	int timer_count, i, ret = 0;
1648
1649	timers = kcalloc(platform_timer_count, sizeof(*timers),
1650			    GFP_KERNEL);
1651	if (!timers)
1652		return -ENOMEM;
1653
1654	ret = acpi_arch_timer_mem_init(timers, &timer_count);
1655	if (ret || !timer_count)
1656		goto out;
1657
1658	/*
1659	 * While unlikely, it's theoretically possible that none of the frames
1660	 * in a timer expose the combination of feature we want.
1661	 */
1662	for (i = 0; i < timer_count; i++) {
1663		timer = &timers[i];
1664
1665		frame = arch_timer_mem_find_best_frame(timer);
1666		if (!best_frame)
1667			best_frame = frame;
1668
1669		ret = arch_timer_mem_verify_cntfrq(timer);
1670		if (ret) {
1671			pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1672			goto out;
1673		}
1674
1675		if (!best_frame) /* implies !frame */
1676			/*
1677			 * Only complain about missing suitable frames if we
1678			 * haven't already found one in a previous iteration.
1679			 */
1680			pr_err("Unable to find a suitable frame in timer @ %pa\n",
1681				&timer->cntctlbase);
1682	}
1683
1684	if (best_frame)
1685		ret = arch_timer_mem_frame_register(best_frame);
1686out:
1687	kfree(timers);
1688	return ret;
1689}
1690
1691/* Initialize per-processor generic timer and memory-mapped timer(if present) */
1692static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1693{
1694	int ret, platform_timer_count;
1695
1696	if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1697		pr_warn("already initialized, skipping\n");
1698		return -EINVAL;
1699	}
1700
1701	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1702
1703	ret = acpi_gtdt_init(table, &platform_timer_count);
1704	if (ret)
1705		return ret;
1706
1707	arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1708		acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1709
1710	arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1711		acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1712
1713	arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1714		acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1715
1716	arch_timer_populate_kvm_info();
1717
1718	/*
1719	 * When probing via ACPI, we have no mechanism to override the sysreg
1720	 * CNTFRQ value. This *must* be correct.
1721	 */
1722	arch_timer_rate = arch_timer_get_cntfrq();
1723	ret = validate_timer_rate();
1724	if (ret) {
1725		pr_err(FW_BUG "frequency not available.\n");
1726		return ret;
1727	}
1728
1729	arch_timer_uses_ppi = arch_timer_select_ppi();
1730	if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1731		pr_err("No interrupt available, giving up\n");
1732		return -EINVAL;
1733	}
1734
1735	/* Always-on capability */
1736	arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1737
1738	/* Check for globally applicable workarounds */
1739	arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1740
1741	ret = arch_timer_register();
1742	if (ret)
1743		return ret;
1744
1745	if (platform_timer_count &&
1746	    arch_timer_mem_acpi_init(platform_timer_count))
1747		pr_err("Failed to initialize memory-mapped timer.\n");
1748
1749	return arch_timer_common_init();
1750}
1751TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1752#endif
1753
1754int kvm_arch_ptp_get_crosststamp(u64 *cycle, struct timespec64 *ts,
1755				 struct clocksource **cs)
1756{
1757	struct arm_smccc_res hvc_res;
1758	u32 ptp_counter;
1759	ktime_t ktime;
1760
1761	if (!IS_ENABLED(CONFIG_HAVE_ARM_SMCCC_DISCOVERY))
1762		return -EOPNOTSUPP;
1763
1764	if (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
1765		ptp_counter = KVM_PTP_VIRT_COUNTER;
1766	else
1767		ptp_counter = KVM_PTP_PHYS_COUNTER;
1768
1769	arm_smccc_1_1_invoke(ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID,
1770			     ptp_counter, &hvc_res);
1771
1772	if ((int)(hvc_res.a0) < 0)
1773		return -EOPNOTSUPP;
1774
1775	ktime = (u64)hvc_res.a0 << 32 | hvc_res.a1;
1776	*ts = ktime_to_timespec64(ktime);
1777	if (cycle)
1778		*cycle = (u64)hvc_res.a2 << 32 | hvc_res.a3;
1779	if (cs)
1780		*cs = &clocksource_counter;
1781
1782	return 0;
1783}
1784EXPORT_SYMBOL_GPL(kvm_arch_ptp_get_crosststamp);