1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/drivers/clocksource/arm_arch_timer.c
   4 *
   5 *  Copyright (C) 2011 ARM Ltd.
   6 *  All Rights Reserved
 
 
 
 
   7 */
   8
   9#define pr_fmt(fmt) 	"arch_timer: " fmt
  10
  11#include <linux/init.h>
  12#include <linux/kernel.h>
  13#include <linux/device.h>
  14#include <linux/smp.h>
  15#include <linux/cpu.h>
  16#include <linux/cpu_pm.h>
  17#include <linux/clockchips.h>
  18#include <linux/clocksource.h>
  19#include <linux/interrupt.h>
  20#include <linux/of_irq.h>
  21#include <linux/of_address.h>
  22#include <linux/io.h>
  23#include <linux/slab.h>
  24#include <linux/sched/clock.h>
  25#include <linux/sched_clock.h>
  26#include <linux/acpi.h>
  27
  28#include <asm/arch_timer.h>
  29#include <asm/virt.h>
  30
  31#include <clocksource/arm_arch_timer.h>
  32
 
 
 
  33#define CNTTIDR		0x08
  34#define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4))
  35
  36#define CNTACR(n)	(0x40 + ((n) * 4))
  37#define CNTACR_RPCT	BIT(0)
  38#define CNTACR_RVCT	BIT(1)
  39#define CNTACR_RFRQ	BIT(2)
  40#define CNTACR_RVOFF	BIT(3)
  41#define CNTACR_RWVT	BIT(4)
  42#define CNTACR_RWPT	BIT(5)
  43
  44#define CNTVCT_LO	0x08
  45#define CNTVCT_HI	0x0c
  46#define CNTFRQ		0x10
  47#define CNTP_TVAL	0x28
  48#define CNTP_CTL	0x2c
  49#define CNTV_TVAL	0x38
  50#define CNTV_CTL	0x3c
  51
  52static unsigned arch_timers_present __initdata;
  53
  54static void __iomem *arch_counter_base;
  55
  56struct arch_timer {
  57	void __iomem *base;
  58	struct clock_event_device evt;
  59};
  60
  61#define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  62
  63static u32 arch_timer_rate;
  64static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI];
  65
  66static struct clock_event_device __percpu *arch_timer_evt;
  67
  68static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI;
  69static bool arch_timer_c3stop;
  70static bool arch_timer_mem_use_virtual;
  71static bool arch_counter_suspend_stop;
  72#ifdef CONFIG_GENERIC_GETTIMEOFDAY
  73static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;
  74#else
  75static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;
  76#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
  77
  78static cpumask_t evtstrm_available = CPU_MASK_NONE;
  79static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  80
  81static int __init early_evtstrm_cfg(char *buf)
  82{
  83	return strtobool(buf, &evtstrm_enable);
  84}
  85early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  86
  87/*
  88 * Architected system timer support.
  89 */
  90
  91static __always_inline
  92void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
  93			  struct clock_event_device *clk)
  94{
  95	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
  96		struct arch_timer *timer = to_arch_timer(clk);
  97		switch (reg) {
  98		case ARCH_TIMER_REG_CTRL:
  99			writel_relaxed(val, timer->base + CNTP_CTL);
 100			break;
 101		case ARCH_TIMER_REG_TVAL:
 102			writel_relaxed(val, timer->base + CNTP_TVAL);
 103			break;
 104		}
 105	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 106		struct arch_timer *timer = to_arch_timer(clk);
 107		switch (reg) {
 108		case ARCH_TIMER_REG_CTRL:
 109			writel_relaxed(val, timer->base + CNTV_CTL);
 110			break;
 111		case ARCH_TIMER_REG_TVAL:
 112			writel_relaxed(val, timer->base + CNTV_TVAL);
 113			break;
 114		}
 115	} else {
 116		arch_timer_reg_write_cp15(access, reg, val);
 117	}
 118}
 119
 120static __always_inline
 121u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 122			struct clock_event_device *clk)
 123{
 124	u32 val;
 125
 126	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 127		struct arch_timer *timer = to_arch_timer(clk);
 128		switch (reg) {
 129		case ARCH_TIMER_REG_CTRL:
 130			val = readl_relaxed(timer->base + CNTP_CTL);
 131			break;
 132		case ARCH_TIMER_REG_TVAL:
 133			val = readl_relaxed(timer->base + CNTP_TVAL);
 134			break;
 135		}
 136	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 137		struct arch_timer *timer = to_arch_timer(clk);
 138		switch (reg) {
 139		case ARCH_TIMER_REG_CTRL:
 140			val = readl_relaxed(timer->base + CNTV_CTL);
 141			break;
 142		case ARCH_TIMER_REG_TVAL:
 143			val = readl_relaxed(timer->base + CNTV_TVAL);
 144			break;
 145		}
 146	} else {
 147		val = arch_timer_reg_read_cp15(access, reg);
 148	}
 149
 150	return val;
 151}
 152
 153static notrace u64 arch_counter_get_cntpct_stable(void)
 154{
 155	return __arch_counter_get_cntpct_stable();
 156}
 157
 158static notrace u64 arch_counter_get_cntpct(void)
 159{
 160	return __arch_counter_get_cntpct();
 161}
 162
 163static notrace u64 arch_counter_get_cntvct_stable(void)
 164{
 165	return __arch_counter_get_cntvct_stable();
 166}
 167
 168static notrace u64 arch_counter_get_cntvct(void)
 169{
 170	return __arch_counter_get_cntvct();
 171}
 172
 173/*
 174 * Default to cp15 based access because arm64 uses this function for
 175 * sched_clock() before DT is probed and the cp15 method is guaranteed
 176 * to exist on arm64. arm doesn't use this before DT is probed so even
 177 * if we don't have the cp15 accessors we won't have a problem.
 178 */
 179u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 180EXPORT_SYMBOL_GPL(arch_timer_read_counter);
 181
 182static u64 arch_counter_read(struct clocksource *cs)
 183{
 184	return arch_timer_read_counter();
 185}
 186
 187static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 188{
 189	return arch_timer_read_counter();
 190}
 191
 192static struct clocksource clocksource_counter = {
 193	.name	= "arch_sys_counter",
 194	.rating	= 400,
 195	.read	= arch_counter_read,
 196	.mask	= CLOCKSOURCE_MASK(56),
 197	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
 198};
 199
 200static struct cyclecounter cyclecounter __ro_after_init = {
 201	.read	= arch_counter_read_cc,
 202	.mask	= CLOCKSOURCE_MASK(56),
 203};
 204
 205struct ate_acpi_oem_info {
 206	char oem_id[ACPI_OEM_ID_SIZE + 1];
 207	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
 208	u32 oem_revision;
 209};
 210
 211#ifdef CONFIG_FSL_ERRATUM_A008585
 212/*
 213 * The number of retries is an arbitrary value well beyond the highest number
 214 * of iterations the loop has been observed to take.
 215 */
 216#define __fsl_a008585_read_reg(reg) ({			\
 217	u64 _old, _new;					\
 218	int _retries = 200;				\
 219							\
 220	do {						\
 221		_old = read_sysreg(reg);		\
 222		_new = read_sysreg(reg);		\
 223		_retries--;				\
 224	} while (unlikely(_old != _new) && _retries);	\
 225							\
 226	WARN_ON_ONCE(!_retries);			\
 227	_new;						\
 228})
 229
 230static u32 notrace fsl_a008585_read_cntp_tval_el0(void)
 231{
 232	return __fsl_a008585_read_reg(cntp_tval_el0);
 233}
 234
 235static u32 notrace fsl_a008585_read_cntv_tval_el0(void)
 236{
 237	return __fsl_a008585_read_reg(cntv_tval_el0);
 238}
 239
 240static u64 notrace fsl_a008585_read_cntpct_el0(void)
 241{
 242	return __fsl_a008585_read_reg(cntpct_el0);
 243}
 244
 245static u64 notrace fsl_a008585_read_cntvct_el0(void)
 246{
 247	return __fsl_a008585_read_reg(cntvct_el0);
 248}
 249#endif
 250
 251#ifdef CONFIG_HISILICON_ERRATUM_161010101
 252/*
 253 * Verify whether the value of the second read is larger than the first by
 254 * less than 32 is the only way to confirm the value is correct, so clear the
 255 * lower 5 bits to check whether the difference is greater than 32 or not.
 256 * Theoretically the erratum should not occur more than twice in succession
 257 * when reading the system counter, but it is possible that some interrupts
 258 * may lead to more than twice read errors, triggering the warning, so setting
 259 * the number of retries far beyond the number of iterations the loop has been
 260 * observed to take.
 261 */
 262#define __hisi_161010101_read_reg(reg) ({				\
 263	u64 _old, _new;						\
 264	int _retries = 50;					\
 265								\
 266	do {							\
 267		_old = read_sysreg(reg);			\
 268		_new = read_sysreg(reg);			\
 269		_retries--;					\
 270	} while (unlikely((_new - _old) >> 5) && _retries);	\
 271								\
 272	WARN_ON_ONCE(!_retries);				\
 273	_new;							\
 274})
 275
 276static u32 notrace hisi_161010101_read_cntp_tval_el0(void)
 277{
 278	return __hisi_161010101_read_reg(cntp_tval_el0);
 279}
 280
 281static u32 notrace hisi_161010101_read_cntv_tval_el0(void)
 282{
 283	return __hisi_161010101_read_reg(cntv_tval_el0);
 284}
 285
 286static u64 notrace hisi_161010101_read_cntpct_el0(void)
 287{
 288	return __hisi_161010101_read_reg(cntpct_el0);
 289}
 290
 291static u64 notrace hisi_161010101_read_cntvct_el0(void)
 292{
 293	return __hisi_161010101_read_reg(cntvct_el0);
 294}
 295
 296static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
 297	/*
 298	 * Note that trailing spaces are required to properly match
 299	 * the OEM table information.
 300	 */
 301	{
 302		.oem_id		= "HISI  ",
 303		.oem_table_id	= "HIP05   ",
 304		.oem_revision	= 0,
 305	},
 306	{
 307		.oem_id		= "HISI  ",
 308		.oem_table_id	= "HIP06   ",
 309		.oem_revision	= 0,
 310	},
 311	{
 312		.oem_id		= "HISI  ",
 313		.oem_table_id	= "HIP07   ",
 314		.oem_revision	= 0,
 315	},
 316	{ /* Sentinel indicating the end of the OEM array */ },
 317};
 318#endif
 319
 320#ifdef CONFIG_ARM64_ERRATUM_858921
 321static u64 notrace arm64_858921_read_cntpct_el0(void)
 322{
 323	u64 old, new;
 324
 325	old = read_sysreg(cntpct_el0);
 326	new = read_sysreg(cntpct_el0);
 327	return (((old ^ new) >> 32) & 1) ? old : new;
 328}
 329
 330static u64 notrace arm64_858921_read_cntvct_el0(void)
 331{
 332	u64 old, new;
 333
 334	old = read_sysreg(cntvct_el0);
 335	new = read_sysreg(cntvct_el0);
 336	return (((old ^ new) >> 32) & 1) ? old : new;
 337}
 338#endif
 339
 340#ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 341/*
 342 * The low bits of the counter registers are indeterminate while bit 10 or
 343 * greater is rolling over. Since the counter value can jump both backward
 344 * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
 345 * with all ones or all zeros in the low bits. Bound the loop by the maximum
 346 * number of CPU cycles in 3 consecutive 24 MHz counter periods.
 347 */
 348#define __sun50i_a64_read_reg(reg) ({					\
 349	u64 _val;							\
 350	int _retries = 150;						\
 351									\
 352	do {								\
 353		_val = read_sysreg(reg);				\
 354		_retries--;						\
 355	} while (((_val + 1) & GENMASK(9, 0)) <= 1 && _retries);	\
 356									\
 357	WARN_ON_ONCE(!_retries);					\
 358	_val;								\
 359})
 360
 361static u64 notrace sun50i_a64_read_cntpct_el0(void)
 362{
 363	return __sun50i_a64_read_reg(cntpct_el0);
 364}
 365
 366static u64 notrace sun50i_a64_read_cntvct_el0(void)
 367{
 368	return __sun50i_a64_read_reg(cntvct_el0);
 369}
 370
 371static u32 notrace sun50i_a64_read_cntp_tval_el0(void)
 372{
 373	return read_sysreg(cntp_cval_el0) - sun50i_a64_read_cntpct_el0();
 374}
 375
 376static u32 notrace sun50i_a64_read_cntv_tval_el0(void)
 377{
 378	return read_sysreg(cntv_cval_el0) - sun50i_a64_read_cntvct_el0();
 379}
 380#endif
 381
 382#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
 383DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
 384EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
 385
 386static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
 
 387
 388static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,
 389						struct clock_event_device *clk)
 390{
 391	unsigned long ctrl;
 392	u64 cval;
 393
 394	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 395	ctrl |= ARCH_TIMER_CTRL_ENABLE;
 396	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 397
 398	if (access == ARCH_TIMER_PHYS_ACCESS) {
 399		cval = evt + arch_counter_get_cntpct();
 400		write_sysreg(cval, cntp_cval_el0);
 401	} else {
 402		cval = evt + arch_counter_get_cntvct();
 403		write_sysreg(cval, cntv_cval_el0);
 404	}
 405
 406	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 407}
 408
 409static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt,
 410					    struct clock_event_device *clk)
 411{
 412	erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 413	return 0;
 414}
 415
 416static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt,
 417					    struct clock_event_device *clk)
 418{
 419	erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 420	return 0;
 421}
 422
 423static const struct arch_timer_erratum_workaround ool_workarounds[] = {
 424#ifdef CONFIG_FSL_ERRATUM_A008585
 425	{
 426		.match_type = ate_match_dt,
 427		.id = "fsl,erratum-a008585",
 428		.desc = "Freescale erratum a005858",
 429		.read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,
 430		.read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,
 431		.read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
 432		.read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
 433		.set_next_event_phys = erratum_set_next_event_tval_phys,
 434		.set_next_event_virt = erratum_set_next_event_tval_virt,
 435	},
 436#endif
 437#ifdef CONFIG_HISILICON_ERRATUM_161010101
 438	{
 439		.match_type = ate_match_dt,
 440		.id = "hisilicon,erratum-161010101",
 441		.desc = "HiSilicon erratum 161010101",
 442		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
 443		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
 444		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 445		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 446		.set_next_event_phys = erratum_set_next_event_tval_phys,
 447		.set_next_event_virt = erratum_set_next_event_tval_virt,
 448	},
 449	{
 450		.match_type = ate_match_acpi_oem_info,
 451		.id = hisi_161010101_oem_info,
 452		.desc = "HiSilicon erratum 161010101",
 453		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
 454		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
 455		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 456		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 457		.set_next_event_phys = erratum_set_next_event_tval_phys,
 458		.set_next_event_virt = erratum_set_next_event_tval_virt,
 459	},
 460#endif
 461#ifdef CONFIG_ARM64_ERRATUM_858921
 462	{
 463		.match_type = ate_match_local_cap_id,
 464		.id = (void *)ARM64_WORKAROUND_858921,
 465		.desc = "ARM erratum 858921",
 466		.read_cntpct_el0 = arm64_858921_read_cntpct_el0,
 467		.read_cntvct_el0 = arm64_858921_read_cntvct_el0,
 468	},
 469#endif
 470#ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 471	{
 472		.match_type = ate_match_dt,
 473		.id = "allwinner,erratum-unknown1",
 474		.desc = "Allwinner erratum UNKNOWN1",
 475		.read_cntp_tval_el0 = sun50i_a64_read_cntp_tval_el0,
 476		.read_cntv_tval_el0 = sun50i_a64_read_cntv_tval_el0,
 477		.read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
 478		.read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
 479		.set_next_event_phys = erratum_set_next_event_tval_phys,
 480		.set_next_event_virt = erratum_set_next_event_tval_virt,
 481	},
 482#endif
 483#ifdef CONFIG_ARM64_ERRATUM_1418040
 484	{
 485		.match_type = ate_match_local_cap_id,
 486		.id = (void *)ARM64_WORKAROUND_1418040,
 487		.desc = "ARM erratum 1418040",
 488		.disable_compat_vdso = true,
 489	},
 490#endif
 491};
 492
 493typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
 494			       const void *);
 495
 496static
 497bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
 498				 const void *arg)
 499{
 500	const struct device_node *np = arg;
 501
 502	return of_property_read_bool(np, wa->id);
 503}
 504
 505static
 506bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
 507					const void *arg)
 508{
 509	return this_cpu_has_cap((uintptr_t)wa->id);
 510}
 511
 512
 513static
 514bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
 515				       const void *arg)
 516{
 517	static const struct ate_acpi_oem_info empty_oem_info = {};
 518	const struct ate_acpi_oem_info *info = wa->id;
 519	const struct acpi_table_header *table = arg;
 520
 521	/* Iterate over the ACPI OEM info array, looking for a match */
 522	while (memcmp(info, &empty_oem_info, sizeof(*info))) {
 523		if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
 524		    !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
 525		    info->oem_revision == table->oem_revision)
 526			return true;
 527
 528		info++;
 529	}
 530
 531	return false;
 532}
 533
 534static const struct arch_timer_erratum_workaround *
 535arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
 536			  ate_match_fn_t match_fn,
 537			  void *arg)
 538{
 539	int i;
 540
 541	for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
 542		if (ool_workarounds[i].match_type != type)
 543			continue;
 544
 545		if (match_fn(&ool_workarounds[i], arg))
 546			return &ool_workarounds[i];
 547	}
 548
 549	return NULL;
 550}
 551
 552static
 553void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
 554				  bool local)
 555{
 556	int i;
 557
 558	if (local) {
 559		__this_cpu_write(timer_unstable_counter_workaround, wa);
 560	} else {
 561		for_each_possible_cpu(i)
 562			per_cpu(timer_unstable_counter_workaround, i) = wa;
 563	}
 564
 565	if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
 566		atomic_set(&timer_unstable_counter_workaround_in_use, 1);
 
 
 
 567
 568	/*
 569	 * Don't use the vdso fastpath if errata require using the
 570	 * out-of-line counter accessor. We may change our mind pretty
 571	 * late in the game (with a per-CPU erratum, for example), so
 572	 * change both the default value and the vdso itself.
 573	 */
 574	if (wa->read_cntvct_el0) {
 575		clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
 576		vdso_default = VDSO_CLOCKMODE_NONE;
 577	} else if (wa->disable_compat_vdso && vdso_default != VDSO_CLOCKMODE_NONE) {
 578		vdso_default = VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT;
 579		clocksource_counter.vdso_clock_mode = vdso_default;
 580	}
 581}
 582
 583static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
 584					    void *arg)
 585{
 586	const struct arch_timer_erratum_workaround *wa, *__wa;
 587	ate_match_fn_t match_fn = NULL;
 588	bool local = false;
 589
 590	switch (type) {
 591	case ate_match_dt:
 592		match_fn = arch_timer_check_dt_erratum;
 593		break;
 594	case ate_match_local_cap_id:
 595		match_fn = arch_timer_check_local_cap_erratum;
 596		local = true;
 597		break;
 598	case ate_match_acpi_oem_info:
 599		match_fn = arch_timer_check_acpi_oem_erratum;
 600		break;
 601	default:
 602		WARN_ON(1);
 603		return;
 604	}
 605
 606	wa = arch_timer_iterate_errata(type, match_fn, arg);
 607	if (!wa)
 608		return;
 609
 610	__wa = __this_cpu_read(timer_unstable_counter_workaround);
 611	if (__wa && wa != __wa)
 612		pr_warn("Can't enable workaround for %s (clashes with %s\n)",
 613			wa->desc, __wa->desc);
 
 
 614
 615	if (__wa)
 616		return;
 
 617
 618	arch_timer_enable_workaround(wa, local);
 619	pr_info("Enabling %s workaround for %s\n",
 620		local ? "local" : "global", wa->desc);
 621}
 622
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 623static bool arch_timer_this_cpu_has_cntvct_wa(void)
 624{
 625	return has_erratum_handler(read_cntvct_el0);
 626}
 627
 628static bool arch_timer_counter_has_wa(void)
 629{
 630	return atomic_read(&timer_unstable_counter_workaround_in_use);
 631}
 632#else
 633#define arch_timer_check_ool_workaround(t,a)		do { } while(0)
 
 
 
 634#define arch_timer_this_cpu_has_cntvct_wa()		({false;})
 635#define arch_timer_counter_has_wa()			({false;})
 636#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
 637
 638static __always_inline irqreturn_t timer_handler(const int access,
 639					struct clock_event_device *evt)
 640{
 641	unsigned long ctrl;
 642
 643	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 644	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 645		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 646		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 647		evt->event_handler(evt);
 648		return IRQ_HANDLED;
 649	}
 650
 651	return IRQ_NONE;
 652}
 653
 654static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 655{
 656	struct clock_event_device *evt = dev_id;
 657
 658	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 659}
 660
 661static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 662{
 663	struct clock_event_device *evt = dev_id;
 664
 665	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 666}
 667
 668static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 669{
 670	struct clock_event_device *evt = dev_id;
 671
 672	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 673}
 674
 675static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 676{
 677	struct clock_event_device *evt = dev_id;
 678
 679	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 680}
 681
 682static __always_inline int timer_shutdown(const int access,
 683					  struct clock_event_device *clk)
 684{
 685	unsigned long ctrl;
 686
 687	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 688	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 689	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 690
 691	return 0;
 692}
 693
 694static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 695{
 696	return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 697}
 698
 699static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 700{
 701	return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 702}
 703
 704static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 705{
 706	return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 707}
 708
 709static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 710{
 711	return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 712}
 713
 714static __always_inline void set_next_event(const int access, unsigned long evt,
 715					   struct clock_event_device *clk)
 716{
 717	unsigned long ctrl;
 718	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 719	ctrl |= ARCH_TIMER_CTRL_ENABLE;
 720	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 721	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 722	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 723}
 724
 725static int arch_timer_set_next_event_virt(unsigned long evt,
 726					  struct clock_event_device *clk)
 727{
 
 
 
 
 
 728	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 729	return 0;
 730}
 731
 732static int arch_timer_set_next_event_phys(unsigned long evt,
 733					  struct clock_event_device *clk)
 734{
 
 
 
 
 
 735	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 736	return 0;
 737}
 738
 739static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 740					      struct clock_event_device *clk)
 741{
 742	set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 743	return 0;
 744}
 745
 746static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 747					      struct clock_event_device *clk)
 748{
 749	set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 750	return 0;
 751}
 752
 753static void __arch_timer_setup(unsigned type,
 754			       struct clock_event_device *clk)
 755{
 756	clk->features = CLOCK_EVT_FEAT_ONESHOT;
 757
 758	if (type == ARCH_TIMER_TYPE_CP15) {
 759		typeof(clk->set_next_event) sne;
 760
 761		arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
 762
 763		if (arch_timer_c3stop)
 764			clk->features |= CLOCK_EVT_FEAT_C3STOP;
 765		clk->name = "arch_sys_timer";
 766		clk->rating = 450;
 767		clk->cpumask = cpumask_of(smp_processor_id());
 768		clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 769		switch (arch_timer_uses_ppi) {
 770		case ARCH_TIMER_VIRT_PPI:
 771			clk->set_state_shutdown = arch_timer_shutdown_virt;
 772			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 773			sne = erratum_handler(set_next_event_virt);
 774			break;
 775		case ARCH_TIMER_PHYS_SECURE_PPI:
 776		case ARCH_TIMER_PHYS_NONSECURE_PPI:
 777		case ARCH_TIMER_HYP_PPI:
 778			clk->set_state_shutdown = arch_timer_shutdown_phys;
 779			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 780			sne = erratum_handler(set_next_event_phys);
 781			break;
 782		default:
 783			BUG();
 784		}
 785
 786		clk->set_next_event = sne;
 787	} else {
 788		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 789		clk->name = "arch_mem_timer";
 790		clk->rating = 400;
 791		clk->cpumask = cpu_possible_mask;
 792		if (arch_timer_mem_use_virtual) {
 793			clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 794			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 795			clk->set_next_event =
 796				arch_timer_set_next_event_virt_mem;
 797		} else {
 798			clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 799			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 800			clk->set_next_event =
 801				arch_timer_set_next_event_phys_mem;
 802		}
 803	}
 804
 805	clk->set_state_shutdown(clk);
 806
 807	clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 808}
 809
 810static void arch_timer_evtstrm_enable(int divider)
 811{
 812	u32 cntkctl = arch_timer_get_cntkctl();
 813
 814	cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 815	/* Set the divider and enable virtual event stream */
 816	cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 817			| ARCH_TIMER_VIRT_EVT_EN;
 818	arch_timer_set_cntkctl(cntkctl);
 819	arch_timer_set_evtstrm_feature();
 
 
 
 820	cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
 821}
 822
 823static void arch_timer_configure_evtstream(void)
 824{
 825	int evt_stream_div, pos;
 826
 827	/* Find the closest power of two to the divisor */
 828	evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 829	pos = fls(evt_stream_div);
 830	if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 831		pos--;
 832	/* enable event stream */
 833	arch_timer_evtstrm_enable(min(pos, 15));
 834}
 835
 836static void arch_counter_set_user_access(void)
 837{
 838	u32 cntkctl = arch_timer_get_cntkctl();
 839
 840	/* Disable user access to the timers and both counters */
 841	/* Also disable virtual event stream */
 842	cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 843			| ARCH_TIMER_USR_VT_ACCESS_EN
 844		        | ARCH_TIMER_USR_VCT_ACCESS_EN
 845			| ARCH_TIMER_VIRT_EVT_EN
 846			| ARCH_TIMER_USR_PCT_ACCESS_EN);
 847
 848	/*
 849	 * Enable user access to the virtual counter if it doesn't
 850	 * need to be workaround. The vdso may have been already
 851	 * disabled though.
 852	 */
 853	if (arch_timer_this_cpu_has_cntvct_wa())
 854		pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
 855	else
 856		cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 857
 858	arch_timer_set_cntkctl(cntkctl);
 859}
 860
 861static bool arch_timer_has_nonsecure_ppi(void)
 862{
 863	return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
 864		arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 865}
 866
 867static u32 check_ppi_trigger(int irq)
 868{
 869	u32 flags = irq_get_trigger_type(irq);
 870
 871	if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 872		pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 873		pr_warn("WARNING: Please fix your firmware\n");
 874		flags = IRQF_TRIGGER_LOW;
 875	}
 876
 877	return flags;
 878}
 879
 880static int arch_timer_starting_cpu(unsigned int cpu)
 881{
 882	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 883	u32 flags;
 884
 885	__arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
 886
 887	flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 888	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 889
 890	if (arch_timer_has_nonsecure_ppi()) {
 891		flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 892		enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
 893				  flags);
 894	}
 895
 896	arch_counter_set_user_access();
 897	if (evtstrm_enable)
 898		arch_timer_configure_evtstream();
 899
 900	return 0;
 901}
 902
 903static int validate_timer_rate(void)
 904{
 905	if (!arch_timer_rate)
 906		return -EINVAL;
 907
 908	/* Arch timer frequency < 1MHz can cause trouble */
 909	WARN_ON(arch_timer_rate < 1000000);
 910
 911	return 0;
 912}
 913
 914/*
 915 * For historical reasons, when probing with DT we use whichever (non-zero)
 916 * rate was probed first, and don't verify that others match. If the first node
 917 * probed has a clock-frequency property, this overrides the HW register.
 918 */
 919static void arch_timer_of_configure_rate(u32 rate, struct device_node *np)
 920{
 921	/* Who has more than one independent system counter? */
 922	if (arch_timer_rate)
 923		return;
 924
 925	if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
 926		arch_timer_rate = rate;
 927
 928	/* Check the timer frequency. */
 929	if (validate_timer_rate())
 930		pr_warn("frequency not available\n");
 931}
 932
 933static void arch_timer_banner(unsigned type)
 934{
 935	pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 936		type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
 937		type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
 938			" and " : "",
 939		type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
 940		(unsigned long)arch_timer_rate / 1000000,
 941		(unsigned long)(arch_timer_rate / 10000) % 100,
 942		type & ARCH_TIMER_TYPE_CP15 ?
 943			(arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
 944			"",
 945		type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
 946		type & ARCH_TIMER_TYPE_MEM ?
 947			arch_timer_mem_use_virtual ? "virt" : "phys" :
 948			"");
 949}
 950
 951u32 arch_timer_get_rate(void)
 952{
 953	return arch_timer_rate;
 954}
 955
 956bool arch_timer_evtstrm_available(void)
 957{
 958	/*
 959	 * We might get called from a preemptible context. This is fine
 960	 * because availability of the event stream should be always the same
 961	 * for a preemptible context and context where we might resume a task.
 962	 */
 963	return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
 964}
 965
 966static u64 arch_counter_get_cntvct_mem(void)
 967{
 968	u32 vct_lo, vct_hi, tmp_hi;
 969
 970	do {
 971		vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 972		vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 973		tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 974	} while (vct_hi != tmp_hi);
 975
 976	return ((u64) vct_hi << 32) | vct_lo;
 977}
 978
 979static struct arch_timer_kvm_info arch_timer_kvm_info;
 980
 981struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 982{
 983	return &arch_timer_kvm_info;
 984}
 985
 986static void __init arch_counter_register(unsigned type)
 987{
 988	u64 start_count;
 989
 990	/* Register the CP15 based counter if we have one */
 991	if (type & ARCH_TIMER_TYPE_CP15) {
 992		u64 (*rd)(void);
 993
 994		if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
 995		    arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
 996			if (arch_timer_counter_has_wa())
 997				rd = arch_counter_get_cntvct_stable;
 998			else
 999				rd = arch_counter_get_cntvct;
1000		} else {
1001			if (arch_timer_counter_has_wa())
1002				rd = arch_counter_get_cntpct_stable;
1003			else
1004				rd = arch_counter_get_cntpct;
1005		}
1006
1007		arch_timer_read_counter = rd;
1008		clocksource_counter.vdso_clock_mode = vdso_default;
1009	} else {
1010		arch_timer_read_counter = arch_counter_get_cntvct_mem;
1011	}
1012
1013	if (!arch_counter_suspend_stop)
1014		clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
1015	start_count = arch_timer_read_counter();
1016	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
1017	cyclecounter.mult = clocksource_counter.mult;
1018	cyclecounter.shift = clocksource_counter.shift;
1019	timecounter_init(&arch_timer_kvm_info.timecounter,
1020			 &cyclecounter, start_count);
1021
1022	/* 56 bits minimum, so we assume worst case rollover */
1023	sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
1024}
1025
1026static void arch_timer_stop(struct clock_event_device *clk)
1027{
1028	pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1029
1030	disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
1031	if (arch_timer_has_nonsecure_ppi())
1032		disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1033
1034	clk->set_state_shutdown(clk);
1035}
1036
1037static int arch_timer_dying_cpu(unsigned int cpu)
1038{
1039	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1040
1041	cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1042
1043	arch_timer_stop(clk);
1044	return 0;
1045}
1046
1047#ifdef CONFIG_CPU_PM
1048static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
1049static int arch_timer_cpu_pm_notify(struct notifier_block *self,
1050				    unsigned long action, void *hcpu)
1051{
1052	if (action == CPU_PM_ENTER) {
1053		__this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1054
1055		cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1056	} else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1057		arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1058
1059		if (arch_timer_have_evtstrm_feature())
1060			cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
1061	}
1062	return NOTIFY_OK;
1063}
1064
1065static struct notifier_block arch_timer_cpu_pm_notifier = {
1066	.notifier_call = arch_timer_cpu_pm_notify,
1067};
1068
1069static int __init arch_timer_cpu_pm_init(void)
1070{
1071	return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
1072}
1073
1074static void __init arch_timer_cpu_pm_deinit(void)
1075{
1076	WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
1077}
1078
1079#else
1080static int __init arch_timer_cpu_pm_init(void)
1081{
1082	return 0;
1083}
1084
1085static void __init arch_timer_cpu_pm_deinit(void)
1086{
1087}
1088#endif
1089
1090static int __init arch_timer_register(void)
1091{
1092	int err;
1093	int ppi;
1094
1095	arch_timer_evt = alloc_percpu(struct clock_event_device);
1096	if (!arch_timer_evt) {
1097		err = -ENOMEM;
1098		goto out;
1099	}
1100
1101	ppi = arch_timer_ppi[arch_timer_uses_ppi];
1102	switch (arch_timer_uses_ppi) {
1103	case ARCH_TIMER_VIRT_PPI:
1104		err = request_percpu_irq(ppi, arch_timer_handler_virt,
1105					 "arch_timer", arch_timer_evt);
1106		break;
1107	case ARCH_TIMER_PHYS_SECURE_PPI:
1108	case ARCH_TIMER_PHYS_NONSECURE_PPI:
1109		err = request_percpu_irq(ppi, arch_timer_handler_phys,
1110					 "arch_timer", arch_timer_evt);
1111		if (!err && arch_timer_has_nonsecure_ppi()) {
1112			ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1113			err = request_percpu_irq(ppi, arch_timer_handler_phys,
1114						 "arch_timer", arch_timer_evt);
1115			if (err)
1116				free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1117						arch_timer_evt);
1118		}
1119		break;
1120	case ARCH_TIMER_HYP_PPI:
1121		err = request_percpu_irq(ppi, arch_timer_handler_phys,
1122					 "arch_timer", arch_timer_evt);
1123		break;
1124	default:
1125		BUG();
1126	}
1127
1128	if (err) {
1129		pr_err("can't register interrupt %d (%d)\n", ppi, err);
1130		goto out_free;
1131	}
1132
1133	err = arch_timer_cpu_pm_init();
1134	if (err)
1135		goto out_unreg_notify;
1136
1137	/* Register and immediately configure the timer on the boot CPU */
1138	err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1139				"clockevents/arm/arch_timer:starting",
1140				arch_timer_starting_cpu, arch_timer_dying_cpu);
1141	if (err)
1142		goto out_unreg_cpupm;
1143	return 0;
1144
1145out_unreg_cpupm:
1146	arch_timer_cpu_pm_deinit();
1147
1148out_unreg_notify:
1149	free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1150	if (arch_timer_has_nonsecure_ppi())
1151		free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1152				arch_timer_evt);
1153
1154out_free:
1155	free_percpu(arch_timer_evt);
1156out:
1157	return err;
1158}
1159
1160static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1161{
1162	int ret;
1163	irq_handler_t func;
1164	struct arch_timer *t;
1165
1166	t = kzalloc(sizeof(*t), GFP_KERNEL);
1167	if (!t)
1168		return -ENOMEM;
1169
1170	t->base = base;
1171	t->evt.irq = irq;
1172	__arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt);
1173
1174	if (arch_timer_mem_use_virtual)
1175		func = arch_timer_handler_virt_mem;
1176	else
1177		func = arch_timer_handler_phys_mem;
1178
1179	ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
1180	if (ret) {
1181		pr_err("Failed to request mem timer irq\n");
1182		kfree(t);
1183	}
1184
1185	return ret;
1186}
1187
1188static const struct of_device_id arch_timer_of_match[] __initconst = {
1189	{ .compatible   = "arm,armv7-timer",    },
1190	{ .compatible   = "arm,armv8-timer",    },
1191	{},
1192};
1193
1194static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1195	{ .compatible   = "arm,armv7-timer-mem", },
1196	{},
1197};
1198
1199static bool __init arch_timer_needs_of_probing(void)
1200{
1201	struct device_node *dn;
1202	bool needs_probing = false;
1203	unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1204
1205	/* We have two timers, and both device-tree nodes are probed. */
1206	if ((arch_timers_present & mask) == mask)
1207		return false;
1208
1209	/*
1210	 * Only one type of timer is probed,
1211	 * check if we have another type of timer node in device-tree.
1212	 */
1213	if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1214		dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1215	else
1216		dn = of_find_matching_node(NULL, arch_timer_of_match);
1217
1218	if (dn && of_device_is_available(dn))
1219		needs_probing = true;
1220
1221	of_node_put(dn);
1222
1223	return needs_probing;
1224}
1225
1226static int __init arch_timer_common_init(void)
1227{
1228	arch_timer_banner(arch_timers_present);
1229	arch_counter_register(arch_timers_present);
1230	return arch_timer_arch_init();
1231}
1232
1233/**
1234 * arch_timer_select_ppi() - Select suitable PPI for the current system.
1235 *
1236 * If HYP mode is available, we know that the physical timer
1237 * has been configured to be accessible from PL1. Use it, so
1238 * that a guest can use the virtual timer instead.
1239 *
1240 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1241 * accesses to CNTP_*_EL1 registers are silently redirected to
1242 * their CNTHP_*_EL2 counterparts, and use a different PPI
1243 * number.
1244 *
1245 * If no interrupt provided for virtual timer, we'll have to
1246 * stick to the physical timer. It'd better be accessible...
1247 * For arm64 we never use the secure interrupt.
1248 *
1249 * Return: a suitable PPI type for the current system.
1250 */
1251static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1252{
1253	if (is_kernel_in_hyp_mode())
1254		return ARCH_TIMER_HYP_PPI;
1255
1256	if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1257		return ARCH_TIMER_VIRT_PPI;
1258
1259	if (IS_ENABLED(CONFIG_ARM64))
1260		return ARCH_TIMER_PHYS_NONSECURE_PPI;
1261
1262	return ARCH_TIMER_PHYS_SECURE_PPI;
1263}
1264
1265static void __init arch_timer_populate_kvm_info(void)
1266{
1267	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1268	if (is_kernel_in_hyp_mode())
1269		arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1270}
1271
1272static int __init arch_timer_of_init(struct device_node *np)
1273{
1274	int i, ret;
1275	u32 rate;
1276
1277	if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1278		pr_warn("multiple nodes in dt, skipping\n");
1279		return 0;
1280	}
1281
1282	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1283	for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++)
1284		arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
1285
1286	arch_timer_populate_kvm_info();
1287
1288	rate = arch_timer_get_cntfrq();
1289	arch_timer_of_configure_rate(rate, np);
1290
1291	arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1292
1293	/* Check for globally applicable workarounds */
1294	arch_timer_check_ool_workaround(ate_match_dt, np);
1295
1296	/*
1297	 * If we cannot rely on firmware initializing the timer registers then
1298	 * we should use the physical timers instead.
1299	 */
1300	if (IS_ENABLED(CONFIG_ARM) &&
1301	    of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1302		arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1303	else
1304		arch_timer_uses_ppi = arch_timer_select_ppi();
1305
1306	if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1307		pr_err("No interrupt available, giving up\n");
1308		return -EINVAL;
1309	}
1310
1311	/* On some systems, the counter stops ticking when in suspend. */
1312	arch_counter_suspend_stop = of_property_read_bool(np,
1313							 "arm,no-tick-in-suspend");
1314
1315	ret = arch_timer_register();
1316	if (ret)
1317		return ret;
1318
1319	if (arch_timer_needs_of_probing())
1320		return 0;
1321
1322	return arch_timer_common_init();
1323}
1324TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1325TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1326
1327static u32 __init
1328arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1329{
1330	void __iomem *base;
1331	u32 rate;
1332
1333	base = ioremap(frame->cntbase, frame->size);
1334	if (!base) {
1335		pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1336		return 0;
1337	}
1338
1339	rate = readl_relaxed(base + CNTFRQ);
1340
1341	iounmap(base);
1342
1343	return rate;
1344}
1345
1346static struct arch_timer_mem_frame * __init
1347arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1348{
1349	struct arch_timer_mem_frame *frame, *best_frame = NULL;
1350	void __iomem *cntctlbase;
1351	u32 cnttidr;
1352	int i;
1353
1354	cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1355	if (!cntctlbase) {
1356		pr_err("Can't map CNTCTLBase @ %pa\n",
1357			&timer_mem->cntctlbase);
1358		return NULL;
1359	}
1360
1361	cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1362
1363	/*
1364	 * Try to find a virtual capable frame. Otherwise fall back to a
1365	 * physical capable frame.
1366	 */
1367	for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1368		u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1369			     CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1370
1371		frame = &timer_mem->frame[i];
1372		if (!frame->valid)
1373			continue;
1374
1375		/* Try enabling everything, and see what sticks */
1376		writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1377		cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1378
1379		if ((cnttidr & CNTTIDR_VIRT(i)) &&
1380		    !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1381			best_frame = frame;
1382			arch_timer_mem_use_virtual = true;
1383			break;
1384		}
1385
1386		if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1387			continue;
1388
1389		best_frame = frame;
1390	}
1391
1392	iounmap(cntctlbase);
1393
1394	return best_frame;
1395}
1396
1397static int __init
1398arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1399{
1400	void __iomem *base;
1401	int ret, irq = 0;
1402
1403	if (arch_timer_mem_use_virtual)
1404		irq = frame->virt_irq;
1405	else
1406		irq = frame->phys_irq;
1407
1408	if (!irq) {
1409		pr_err("Frame missing %s irq.\n",
1410		       arch_timer_mem_use_virtual ? "virt" : "phys");
1411		return -EINVAL;
1412	}
1413
1414	if (!request_mem_region(frame->cntbase, frame->size,
1415				"arch_mem_timer"))
1416		return -EBUSY;
1417
1418	base = ioremap(frame->cntbase, frame->size);
1419	if (!base) {
1420		pr_err("Can't map frame's registers\n");
1421		return -ENXIO;
1422	}
1423
1424	ret = arch_timer_mem_register(base, irq);
1425	if (ret) {
1426		iounmap(base);
1427		return ret;
1428	}
1429
1430	arch_counter_base = base;
1431	arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1432
1433	return 0;
1434}
1435
1436static int __init arch_timer_mem_of_init(struct device_node *np)
1437{
1438	struct arch_timer_mem *timer_mem;
1439	struct arch_timer_mem_frame *frame;
1440	struct device_node *frame_node;
1441	struct resource res;
1442	int ret = -EINVAL;
1443	u32 rate;
1444
1445	timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1446	if (!timer_mem)
1447		return -ENOMEM;
1448
1449	if (of_address_to_resource(np, 0, &res))
1450		goto out;
1451	timer_mem->cntctlbase = res.start;
1452	timer_mem->size = resource_size(&res);
1453
1454	for_each_available_child_of_node(np, frame_node) {
1455		u32 n;
1456		struct arch_timer_mem_frame *frame;
1457
1458		if (of_property_read_u32(frame_node, "frame-number", &n)) {
1459			pr_err(FW_BUG "Missing frame-number.\n");
1460			of_node_put(frame_node);
1461			goto out;
1462		}
1463		if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1464			pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1465			       ARCH_TIMER_MEM_MAX_FRAMES - 1);
1466			of_node_put(frame_node);
1467			goto out;
1468		}
1469		frame = &timer_mem->frame[n];
1470
1471		if (frame->valid) {
1472			pr_err(FW_BUG "Duplicated frame-number.\n");
1473			of_node_put(frame_node);
1474			goto out;
1475		}
1476
1477		if (of_address_to_resource(frame_node, 0, &res)) {
1478			of_node_put(frame_node);
1479			goto out;
1480		}
1481		frame->cntbase = res.start;
1482		frame->size = resource_size(&res);
1483
1484		frame->virt_irq = irq_of_parse_and_map(frame_node,
1485						       ARCH_TIMER_VIRT_SPI);
1486		frame->phys_irq = irq_of_parse_and_map(frame_node,
1487						       ARCH_TIMER_PHYS_SPI);
1488
1489		frame->valid = true;
1490	}
1491
1492	frame = arch_timer_mem_find_best_frame(timer_mem);
1493	if (!frame) {
1494		pr_err("Unable to find a suitable frame in timer @ %pa\n",
1495			&timer_mem->cntctlbase);
1496		ret = -EINVAL;
1497		goto out;
1498	}
1499
1500	rate = arch_timer_mem_frame_get_cntfrq(frame);
1501	arch_timer_of_configure_rate(rate, np);
1502
1503	ret = arch_timer_mem_frame_register(frame);
1504	if (!ret && !arch_timer_needs_of_probing())
1505		ret = arch_timer_common_init();
1506out:
1507	kfree(timer_mem);
1508	return ret;
1509}
1510TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1511		       arch_timer_mem_of_init);
1512
1513#ifdef CONFIG_ACPI_GTDT
1514static int __init
1515arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1516{
1517	struct arch_timer_mem_frame *frame;
1518	u32 rate;
1519	int i;
1520
1521	for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1522		frame = &timer_mem->frame[i];
1523
1524		if (!frame->valid)
1525			continue;
1526
1527		rate = arch_timer_mem_frame_get_cntfrq(frame);
1528		if (rate == arch_timer_rate)
1529			continue;
1530
1531		pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1532			&frame->cntbase,
1533			(unsigned long)rate, (unsigned long)arch_timer_rate);
1534
1535		return -EINVAL;
1536	}
1537
1538	return 0;
1539}
1540
1541static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1542{
1543	struct arch_timer_mem *timers, *timer;
1544	struct arch_timer_mem_frame *frame, *best_frame = NULL;
1545	int timer_count, i, ret = 0;
1546
1547	timers = kcalloc(platform_timer_count, sizeof(*timers),
1548			    GFP_KERNEL);
1549	if (!timers)
1550		return -ENOMEM;
1551
1552	ret = acpi_arch_timer_mem_init(timers, &timer_count);
1553	if (ret || !timer_count)
1554		goto out;
1555
1556	/*
1557	 * While unlikely, it's theoretically possible that none of the frames
1558	 * in a timer expose the combination of feature we want.
1559	 */
1560	for (i = 0; i < timer_count; i++) {
1561		timer = &timers[i];
1562
1563		frame = arch_timer_mem_find_best_frame(timer);
1564		if (!best_frame)
1565			best_frame = frame;
1566
1567		ret = arch_timer_mem_verify_cntfrq(timer);
1568		if (ret) {
1569			pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1570			goto out;
1571		}
1572
1573		if (!best_frame) /* implies !frame */
1574			/*
1575			 * Only complain about missing suitable frames if we
1576			 * haven't already found one in a previous iteration.
1577			 */
1578			pr_err("Unable to find a suitable frame in timer @ %pa\n",
1579				&timer->cntctlbase);
1580	}
1581
1582	if (best_frame)
1583		ret = arch_timer_mem_frame_register(best_frame);
1584out:
1585	kfree(timers);
1586	return ret;
1587}
1588
1589/* Initialize per-processor generic timer and memory-mapped timer(if present) */
1590static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1591{
1592	int ret, platform_timer_count;
1593
1594	if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1595		pr_warn("already initialized, skipping\n");
1596		return -EINVAL;
1597	}
1598
1599	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1600
1601	ret = acpi_gtdt_init(table, &platform_timer_count);
1602	if (ret)
 
1603		return ret;
 
1604
1605	arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1606		acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1607
1608	arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1609		acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1610
1611	arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1612		acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1613
1614	arch_timer_populate_kvm_info();
1615
1616	/*
1617	 * When probing via ACPI, we have no mechanism to override the sysreg
1618	 * CNTFRQ value. This *must* be correct.
1619	 */
1620	arch_timer_rate = arch_timer_get_cntfrq();
1621	ret = validate_timer_rate();
1622	if (ret) {
1623		pr_err(FW_BUG "frequency not available.\n");
1624		return ret;
1625	}
1626
1627	arch_timer_uses_ppi = arch_timer_select_ppi();
1628	if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1629		pr_err("No interrupt available, giving up\n");
1630		return -EINVAL;
1631	}
1632
1633	/* Always-on capability */
1634	arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1635
1636	/* Check for globally applicable workarounds */
1637	arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1638
1639	ret = arch_timer_register();
1640	if (ret)
1641		return ret;
1642
1643	if (platform_timer_count &&
1644	    arch_timer_mem_acpi_init(platform_timer_count))
1645		pr_err("Failed to initialize memory-mapped timer.\n");
1646
1647	return arch_timer_common_init();
1648}
1649TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1650#endif