1/*
   2 *  linux/drivers/clocksource/arm_arch_timer.c
   3 *
   4 *  Copyright (C) 2011 ARM Ltd.
   5 *  All Rights Reserved
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License version 2 as
   9 * published by the Free Software Foundation.
  10 */
  11
  12#define pr_fmt(fmt)	"arm_arch_timer: " fmt
  13
  14#include <linux/init.h>
  15#include <linux/kernel.h>
  16#include <linux/device.h>
  17#include <linux/smp.h>
  18#include <linux/cpu.h>
  19#include <linux/cpu_pm.h>
  20#include <linux/clockchips.h>
  21#include <linux/clocksource.h>
  22#include <linux/interrupt.h>
  23#include <linux/of_irq.h>
  24#include <linux/of_address.h>
  25#include <linux/io.h>
  26#include <linux/slab.h>
 
  27#include <linux/sched_clock.h>
  28#include <linux/acpi.h>
  29
  30#include <asm/arch_timer.h>
  31#include <asm/virt.h>
  32
  33#include <clocksource/arm_arch_timer.h>
  34
 
 
 
  35#define CNTTIDR		0x08
  36#define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4))
  37
  38#define CNTACR(n)	(0x40 + ((n) * 4))
  39#define CNTACR_RPCT	BIT(0)
  40#define CNTACR_RVCT	BIT(1)
  41#define CNTACR_RFRQ	BIT(2)
  42#define CNTACR_RVOFF	BIT(3)
  43#define CNTACR_RWVT	BIT(4)
  44#define CNTACR_RWPT	BIT(5)
  45
  46#define CNTVCT_LO	0x08
  47#define CNTVCT_HI	0x0c
  48#define CNTFRQ		0x10
  49#define CNTP_TVAL	0x28
  50#define CNTP_CTL	0x2c
  51#define CNTV_TVAL	0x38
  52#define CNTV_CTL	0x3c
  53
  54#define ARCH_CP15_TIMER	BIT(0)
  55#define ARCH_MEM_TIMER	BIT(1)
  56static unsigned arch_timers_present __initdata;
  57
  58static void __iomem *arch_counter_base;
  59
  60struct arch_timer {
  61	void __iomem *base;
  62	struct clock_event_device evt;
  63};
  64
  65#define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  66
  67static u32 arch_timer_rate;
  68
  69enum ppi_nr {
  70	PHYS_SECURE_PPI,
  71	PHYS_NONSECURE_PPI,
  72	VIRT_PPI,
  73	HYP_PPI,
  74	MAX_TIMER_PPI
  75};
  76
  77static int arch_timer_ppi[MAX_TIMER_PPI];
  78
  79static struct clock_event_device __percpu *arch_timer_evt;
  80
  81static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
  82static bool arch_timer_c3stop;
  83static bool arch_timer_mem_use_virtual;
  84static bool arch_counter_suspend_stop;
 
  85
 
  86static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  87
  88static int __init early_evtstrm_cfg(char *buf)
  89{
  90	return strtobool(buf, &evtstrm_enable);
  91}
  92early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  93
  94/*
  95 * Architected system timer support.
  96 */
  97
  98#ifdef CONFIG_FSL_ERRATUM_A008585
  99DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
 100EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
 101
 102static int fsl_a008585_enable = -1;
 103
 104static int __init early_fsl_a008585_cfg(char *buf)
 105{
 106	int ret;
 107	bool val;
 108
 109	ret = strtobool(buf, &val);
 110	if (ret)
 111		return ret;
 112
 113	fsl_a008585_enable = val;
 114	return 0;
 115}
 116early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg);
 117
 118u32 __fsl_a008585_read_cntp_tval_el0(void)
 119{
 120	return __fsl_a008585_read_reg(cntp_tval_el0);
 121}
 122
 123u32 __fsl_a008585_read_cntv_tval_el0(void)
 124{
 125	return __fsl_a008585_read_reg(cntv_tval_el0);
 126}
 127
 128u64 __fsl_a008585_read_cntvct_el0(void)
 129{
 130	return __fsl_a008585_read_reg(cntvct_el0);
 131}
 132EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);
 133#endif /* CONFIG_FSL_ERRATUM_A008585 */
 134
 135static __always_inline
 136void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
 137			  struct clock_event_device *clk)
 138{
 139	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 140		struct arch_timer *timer = to_arch_timer(clk);
 141		switch (reg) {
 142		case ARCH_TIMER_REG_CTRL:
 143			writel_relaxed(val, timer->base + CNTP_CTL);
 144			break;
 145		case ARCH_TIMER_REG_TVAL:
 146			writel_relaxed(val, timer->base + CNTP_TVAL);
 147			break;
 148		}
 149	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 150		struct arch_timer *timer = to_arch_timer(clk);
 151		switch (reg) {
 152		case ARCH_TIMER_REG_CTRL:
 153			writel_relaxed(val, timer->base + CNTV_CTL);
 154			break;
 155		case ARCH_TIMER_REG_TVAL:
 156			writel_relaxed(val, timer->base + CNTV_TVAL);
 157			break;
 158		}
 159	} else {
 160		arch_timer_reg_write_cp15(access, reg, val);
 161	}
 162}
 163
 164static __always_inline
 165u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 166			struct clock_event_device *clk)
 167{
 168	u32 val;
 169
 170	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 171		struct arch_timer *timer = to_arch_timer(clk);
 172		switch (reg) {
 173		case ARCH_TIMER_REG_CTRL:
 174			val = readl_relaxed(timer->base + CNTP_CTL);
 175			break;
 176		case ARCH_TIMER_REG_TVAL:
 177			val = readl_relaxed(timer->base + CNTP_TVAL);
 178			break;
 179		}
 180	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 181		struct arch_timer *timer = to_arch_timer(clk);
 182		switch (reg) {
 183		case ARCH_TIMER_REG_CTRL:
 184			val = readl_relaxed(timer->base + CNTV_CTL);
 185			break;
 186		case ARCH_TIMER_REG_TVAL:
 187			val = readl_relaxed(timer->base + CNTV_TVAL);
 188			break;
 189		}
 190	} else {
 191		val = arch_timer_reg_read_cp15(access, reg);
 192	}
 193
 194	return val;
 195}
 196
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 197static __always_inline irqreturn_t timer_handler(const int access,
 198					struct clock_event_device *evt)
 199{
 200	unsigned long ctrl;
 201
 202	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 203	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 204		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 205		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 206		evt->event_handler(evt);
 207		return IRQ_HANDLED;
 208	}
 209
 210	return IRQ_NONE;
 211}
 212
 213static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 214{
 215	struct clock_event_device *evt = dev_id;
 216
 217	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 218}
 219
 220static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 221{
 222	struct clock_event_device *evt = dev_id;
 223
 224	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 225}
 226
 227static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 228{
 229	struct clock_event_device *evt = dev_id;
 230
 231	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 232}
 233
 234static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 235{
 236	struct clock_event_device *evt = dev_id;
 237
 238	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 239}
 240
 241static __always_inline int timer_shutdown(const int access,
 242					  struct clock_event_device *clk)
 243{
 244	unsigned long ctrl;
 245
 246	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 247	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 248	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 249
 250	return 0;
 251}
 252
 253static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 254{
 255	return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 256}
 257
 258static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 259{
 260	return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 261}
 262
 263static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 264{
 265	return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 266}
 267
 268static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 269{
 270	return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 271}
 272
 273static __always_inline void set_next_event(const int access, unsigned long evt,
 274					   struct clock_event_device *clk)
 275{
 276	unsigned long ctrl;
 277	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 278	ctrl |= ARCH_TIMER_CTRL_ENABLE;
 279	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 280	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 281	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 282}
 283
 284#ifdef CONFIG_FSL_ERRATUM_A008585
 285static __always_inline void fsl_a008585_set_next_event(const int access,
 286		unsigned long evt, struct clock_event_device *clk)
 287{
 288	unsigned long ctrl;
 289	u64 cval = evt + arch_counter_get_cntvct();
 290
 291	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 292	ctrl |= ARCH_TIMER_CTRL_ENABLE;
 293	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 294
 295	if (access == ARCH_TIMER_PHYS_ACCESS)
 296		write_sysreg(cval, cntp_cval_el0);
 297	else if (access == ARCH_TIMER_VIRT_ACCESS)
 298		write_sysreg(cval, cntv_cval_el0);
 299
 300	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 301}
 302
 303static int fsl_a008585_set_next_event_virt(unsigned long evt,
 304					   struct clock_event_device *clk)
 305{
 306	fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 307	return 0;
 308}
 309
 310static int fsl_a008585_set_next_event_phys(unsigned long evt,
 311					   struct clock_event_device *clk)
 312{
 313	fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 314	return 0;
 315}
 316#endif /* CONFIG_FSL_ERRATUM_A008585 */
 317
 318static int arch_timer_set_next_event_virt(unsigned long evt,
 319					  struct clock_event_device *clk)
 320{
 
 
 
 
 
 321	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 322	return 0;
 323}
 324
 325static int arch_timer_set_next_event_phys(unsigned long evt,
 326					  struct clock_event_device *clk)
 327{
 
 
 
 
 
 328	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 329	return 0;
 330}
 331
 332static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 333					      struct clock_event_device *clk)
 334{
 335	set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 336	return 0;
 337}
 338
 339static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 340					      struct clock_event_device *clk)
 341{
 342	set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 343	return 0;
 344}
 345
 346static void fsl_a008585_set_sne(struct clock_event_device *clk)
 347{
 348#ifdef CONFIG_FSL_ERRATUM_A008585
 349	if (!static_branch_unlikely(&arch_timer_read_ool_enabled))
 350		return;
 351
 352	if (arch_timer_uses_ppi == VIRT_PPI)
 353		clk->set_next_event = fsl_a008585_set_next_event_virt;
 354	else
 355		clk->set_next_event = fsl_a008585_set_next_event_phys;
 356#endif
 357}
 358
 359static void __arch_timer_setup(unsigned type,
 360			       struct clock_event_device *clk)
 361{
 362	clk->features = CLOCK_EVT_FEAT_ONESHOT;
 363
 364	if (type == ARCH_CP15_TIMER) {
 365		if (arch_timer_c3stop)
 366			clk->features |= CLOCK_EVT_FEAT_C3STOP;
 367		clk->name = "arch_sys_timer";
 368		clk->rating = 450;
 369		clk->cpumask = cpumask_of(smp_processor_id());
 370		clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 371		switch (arch_timer_uses_ppi) {
 372		case VIRT_PPI:
 373			clk->set_state_shutdown = arch_timer_shutdown_virt;
 374			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 375			clk->set_next_event = arch_timer_set_next_event_virt;
 376			break;
 377		case PHYS_SECURE_PPI:
 378		case PHYS_NONSECURE_PPI:
 379		case HYP_PPI:
 380			clk->set_state_shutdown = arch_timer_shutdown_phys;
 381			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 382			clk->set_next_event = arch_timer_set_next_event_phys;
 383			break;
 384		default:
 385			BUG();
 386		}
 387
 388		fsl_a008585_set_sne(clk);
 389	} else {
 390		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 391		clk->name = "arch_mem_timer";
 392		clk->rating = 400;
 393		clk->cpumask = cpu_all_mask;
 394		if (arch_timer_mem_use_virtual) {
 395			clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 396			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 397			clk->set_next_event =
 398				arch_timer_set_next_event_virt_mem;
 399		} else {
 400			clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 401			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 402			clk->set_next_event =
 403				arch_timer_set_next_event_phys_mem;
 404		}
 405	}
 406
 407	clk->set_state_shutdown(clk);
 408
 409	clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 410}
 411
 412static void arch_timer_evtstrm_enable(int divider)
 413{
 414	u32 cntkctl = arch_timer_get_cntkctl();
 415
 416	cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 417	/* Set the divider and enable virtual event stream */
 418	cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 419			| ARCH_TIMER_VIRT_EVT_EN;
 420	arch_timer_set_cntkctl(cntkctl);
 421	elf_hwcap |= HWCAP_EVTSTRM;
 422#ifdef CONFIG_COMPAT
 423	compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 424#endif
 
 425}
 426
 427static void arch_timer_configure_evtstream(void)
 428{
 429	int evt_stream_div, pos;
 430
 431	/* Find the closest power of two to the divisor */
 432	evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 433	pos = fls(evt_stream_div);
 434	if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 435		pos--;
 436	/* enable event stream */
 437	arch_timer_evtstrm_enable(min(pos, 15));
 438}
 439
 440static void arch_counter_set_user_access(void)
 441{
 442	u32 cntkctl = arch_timer_get_cntkctl();
 443
 444	/* Disable user access to the timers and the physical counter */
 445	/* Also disable virtual event stream */
 446	cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 447			| ARCH_TIMER_USR_VT_ACCESS_EN
 
 448			| ARCH_TIMER_VIRT_EVT_EN
 449			| ARCH_TIMER_USR_PCT_ACCESS_EN);
 450
 451	/* Enable user access to the virtual counter */
 452	cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 
 
 
 
 
 
 
 453
 454	arch_timer_set_cntkctl(cntkctl);
 455}
 456
 457static bool arch_timer_has_nonsecure_ppi(void)
 458{
 459	return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
 460		arch_timer_ppi[PHYS_NONSECURE_PPI]);
 461}
 462
 463static u32 check_ppi_trigger(int irq)
 464{
 465	u32 flags = irq_get_trigger_type(irq);
 466
 467	if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 468		pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 469		pr_warn("WARNING: Please fix your firmware\n");
 470		flags = IRQF_TRIGGER_LOW;
 471	}
 472
 473	return flags;
 474}
 475
 476static int arch_timer_starting_cpu(unsigned int cpu)
 477{
 478	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 479	u32 flags;
 480
 481	__arch_timer_setup(ARCH_CP15_TIMER, clk);
 482
 483	flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 484	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 485
 486	if (arch_timer_has_nonsecure_ppi()) {
 487		flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 488		enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
 
 489	}
 490
 491	arch_counter_set_user_access();
 492	if (evtstrm_enable)
 493		arch_timer_configure_evtstream();
 494
 495	return 0;
 496}
 497
 498static void
 499arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 
 
 
 
 500{
 501	/* Who has more than one independent system counter? */
 502	if (arch_timer_rate)
 503		return;
 504
 505	/*
 506	 * Try to determine the frequency from the device tree or CNTFRQ,
 507	 * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 508	 */
 509	if (!acpi_disabled ||
 510	    of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 511		if (cntbase)
 512			arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 513		else
 514			arch_timer_rate = arch_timer_get_cntfrq();
 515	}
 516
 517	/* Check the timer frequency. */
 518	if (arch_timer_rate == 0)
 519		pr_warn("Architected timer frequency not available\n");
 520}
 521
 522static void arch_timer_banner(unsigned type)
 523{
 524	pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 525		     type & ARCH_CP15_TIMER ? "cp15" : "",
 526		     type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 527		     type & ARCH_MEM_TIMER ? "mmio" : "",
 528		     (unsigned long)arch_timer_rate / 1000000,
 529		     (unsigned long)(arch_timer_rate / 10000) % 100,
 530		     type & ARCH_CP15_TIMER ?
 531		     (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
 
 532			"",
 533		     type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 534		     type & ARCH_MEM_TIMER ?
 535			arch_timer_mem_use_virtual ? "virt" : "phys" :
 536			"");
 537}
 538
 539u32 arch_timer_get_rate(void)
 540{
 541	return arch_timer_rate;
 542}
 543
 
 
 
 
 
 
 
 
 
 
 544static u64 arch_counter_get_cntvct_mem(void)
 545{
 546	u32 vct_lo, vct_hi, tmp_hi;
 547
 548	do {
 549		vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 550		vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 551		tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 552	} while (vct_hi != tmp_hi);
 553
 554	return ((u64) vct_hi << 32) | vct_lo;
 555}
 556
 557/*
 558 * Default to cp15 based access because arm64 uses this function for
 559 * sched_clock() before DT is probed and the cp15 method is guaranteed
 560 * to exist on arm64. arm doesn't use this before DT is probed so even
 561 * if we don't have the cp15 accessors we won't have a problem.
 562 */
 563u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 564
 565static u64 arch_counter_read(struct clocksource *cs)
 566{
 567	return arch_timer_read_counter();
 568}
 569
 570static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 571{
 572	return arch_timer_read_counter();
 573}
 574
 575static struct clocksource clocksource_counter = {
 576	.name	= "arch_sys_counter",
 577	.rating	= 400,
 578	.read	= arch_counter_read,
 579	.mask	= CLOCKSOURCE_MASK(56),
 580	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
 581};
 582
 583static struct cyclecounter cyclecounter = {
 584	.read	= arch_counter_read_cc,
 585	.mask	= CLOCKSOURCE_MASK(56),
 586};
 587
 588static struct arch_timer_kvm_info arch_timer_kvm_info;
 589
 590struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 591{
 592	return &arch_timer_kvm_info;
 593}
 594
 595static void __init arch_counter_register(unsigned type)
 596{
 597	u64 start_count;
 598
 599	/* Register the CP15 based counter if we have one */
 600	if (type & ARCH_CP15_TIMER) {
 601		if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
 
 602			arch_timer_read_counter = arch_counter_get_cntvct;
 603		else
 604			arch_timer_read_counter = arch_counter_get_cntpct;
 605
 606		clocksource_counter.archdata.vdso_direct = true;
 607
 608#ifdef CONFIG_FSL_ERRATUM_A008585
 609		/*
 610		 * Don't use the vdso fastpath if errata require using
 611		 * the out-of-line counter accessor.
 612		 */
 613		if (static_branch_unlikely(&arch_timer_read_ool_enabled))
 614			clocksource_counter.archdata.vdso_direct = false;
 615#endif
 616	} else {
 617		arch_timer_read_counter = arch_counter_get_cntvct_mem;
 618	}
 619
 620	if (!arch_counter_suspend_stop)
 621		clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
 622	start_count = arch_timer_read_counter();
 623	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 624	cyclecounter.mult = clocksource_counter.mult;
 625	cyclecounter.shift = clocksource_counter.shift;
 626	timecounter_init(&arch_timer_kvm_info.timecounter,
 627			 &cyclecounter, start_count);
 628
 629	/* 56 bits minimum, so we assume worst case rollover */
 630	sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 631}
 632
 633static void arch_timer_stop(struct clock_event_device *clk)
 634{
 635	pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 636		 clk->irq, smp_processor_id());
 637
 638	disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
 639	if (arch_timer_has_nonsecure_ppi())
 640		disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 641
 642	clk->set_state_shutdown(clk);
 643}
 644
 645static int arch_timer_dying_cpu(unsigned int cpu)
 646{
 647	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 648
 
 
 649	arch_timer_stop(clk);
 650	return 0;
 651}
 652
 653#ifdef CONFIG_CPU_PM
 654static unsigned int saved_cntkctl;
 655static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 656				    unsigned long action, void *hcpu)
 657{
 658	if (action == CPU_PM_ENTER)
 659		saved_cntkctl = arch_timer_get_cntkctl();
 660	else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 661		arch_timer_set_cntkctl(saved_cntkctl);
 
 
 
 
 
 
 662	return NOTIFY_OK;
 663}
 664
 665static struct notifier_block arch_timer_cpu_pm_notifier = {
 666	.notifier_call = arch_timer_cpu_pm_notify,
 667};
 668
 669static int __init arch_timer_cpu_pm_init(void)
 670{
 671	return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 672}
 673
 674static void __init arch_timer_cpu_pm_deinit(void)
 675{
 676	WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
 677}
 678
 679#else
 680static int __init arch_timer_cpu_pm_init(void)
 681{
 682	return 0;
 683}
 684
 685static void __init arch_timer_cpu_pm_deinit(void)
 686{
 687}
 688#endif
 689
 690static int __init arch_timer_register(void)
 691{
 692	int err;
 693	int ppi;
 694
 695	arch_timer_evt = alloc_percpu(struct clock_event_device);
 696	if (!arch_timer_evt) {
 697		err = -ENOMEM;
 698		goto out;
 699	}
 700
 701	ppi = arch_timer_ppi[arch_timer_uses_ppi];
 702	switch (arch_timer_uses_ppi) {
 703	case VIRT_PPI:
 704		err = request_percpu_irq(ppi, arch_timer_handler_virt,
 705					 "arch_timer", arch_timer_evt);
 706		break;
 707	case PHYS_SECURE_PPI:
 708	case PHYS_NONSECURE_PPI:
 709		err = request_percpu_irq(ppi, arch_timer_handler_phys,
 710					 "arch_timer", arch_timer_evt);
 711		if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 712			ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 713			err = request_percpu_irq(ppi, arch_timer_handler_phys,
 714						 "arch_timer", arch_timer_evt);
 715			if (err)
 716				free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 717						arch_timer_evt);
 718		}
 719		break;
 720	case HYP_PPI:
 721		err = request_percpu_irq(ppi, arch_timer_handler_phys,
 722					 "arch_timer", arch_timer_evt);
 723		break;
 724	default:
 725		BUG();
 726	}
 727
 728	if (err) {
 729		pr_err("arch_timer: can't register interrupt %d (%d)\n",
 730		       ppi, err);
 731		goto out_free;
 732	}
 733
 734	err = arch_timer_cpu_pm_init();
 735	if (err)
 736		goto out_unreg_notify;
 737
 738
 739	/* Register and immediately configure the timer on the boot CPU */
 740	err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
 741				"clockevents/arm/arch_timer:starting",
 742				arch_timer_starting_cpu, arch_timer_dying_cpu);
 743	if (err)
 744		goto out_unreg_cpupm;
 745	return 0;
 746
 747out_unreg_cpupm:
 748	arch_timer_cpu_pm_deinit();
 749
 750out_unreg_notify:
 751	free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
 752	if (arch_timer_has_nonsecure_ppi())
 753		free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 754				arch_timer_evt);
 755
 756out_free:
 757	free_percpu(arch_timer_evt);
 758out:
 759	return err;
 760}
 761
 762static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 763{
 764	int ret;
 765	irq_handler_t func;
 766	struct arch_timer *t;
 767
 768	t = kzalloc(sizeof(*t), GFP_KERNEL);
 769	if (!t)
 770		return -ENOMEM;
 771
 772	t->base = base;
 773	t->evt.irq = irq;
 774	__arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 775
 776	if (arch_timer_mem_use_virtual)
 777		func = arch_timer_handler_virt_mem;
 778	else
 779		func = arch_timer_handler_phys_mem;
 780
 781	ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 782	if (ret) {
 783		pr_err("arch_timer: Failed to request mem timer irq\n");
 784		kfree(t);
 785	}
 786
 787	return ret;
 788}
 789
 790static const struct of_device_id arch_timer_of_match[] __initconst = {
 791	{ .compatible   = "arm,armv7-timer",    },
 792	{ .compatible   = "arm,armv8-timer",    },
 793	{},
 794};
 795
 796static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 797	{ .compatible   = "arm,armv7-timer-mem", },
 798	{},
 799};
 800
 801static bool __init
 802arch_timer_needs_probing(int type, const struct of_device_id *matches)
 803{
 804	struct device_node *dn;
 805	bool needs_probing = false;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 806
 807	dn = of_find_matching_node(NULL, matches);
 808	if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 809		needs_probing = true;
 
 810	of_node_put(dn);
 811
 812	return needs_probing;
 813}
 814
 815static int __init arch_timer_common_init(void)
 816{
 817	unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 818
 819	/* Wait until both nodes are probed if we have two timers */
 820	if ((arch_timers_present & mask) != mask) {
 821		if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 822			return 0;
 823		if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
 824			return 0;
 825	}
 826
 827	arch_timer_banner(arch_timers_present);
 828	arch_counter_register(arch_timers_present);
 829	return arch_timer_arch_init();
 830}
 831
 832static int __init arch_timer_init(void)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 833{
 834	int ret;
 835	/*
 836	 * If HYP mode is available, we know that the physical timer
 837	 * has been configured to be accessible from PL1. Use it, so
 838	 * that a guest can use the virtual timer instead.
 839	 *
 840	 * If no interrupt provided for virtual timer, we'll have to
 841	 * stick to the physical timer. It'd better be accessible...
 842	 *
 843	 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
 844	 * accesses to CNTP_*_EL1 registers are silently redirected to
 845	 * their CNTHP_*_EL2 counterparts, and use a different PPI
 846	 * number.
 847	 */
 848	if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 849		bool has_ppi;
 850
 851		if (is_kernel_in_hyp_mode()) {
 852			arch_timer_uses_ppi = HYP_PPI;
 853			has_ppi = !!arch_timer_ppi[HYP_PPI];
 854		} else {
 855			arch_timer_uses_ppi = PHYS_SECURE_PPI;
 856			has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
 857				   !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
 858		}
 859
 860		if (!has_ppi) {
 861			pr_warn("arch_timer: No interrupt available, giving up\n");
 862			return -EINVAL;
 863		}
 864	}
 865
 866	ret = arch_timer_register();
 867	if (ret)
 868		return ret;
 869
 870	ret = arch_timer_common_init();
 871	if (ret)
 872		return ret;
 873
 874	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
 875	
 876	return 0;
 877}
 878
 879static int __init arch_timer_of_init(struct device_node *np)
 880{
 881	int i;
 
 882
 883	if (arch_timers_present & ARCH_CP15_TIMER) {
 884		pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 885		return 0;
 886	}
 887
 888	arch_timers_present |= ARCH_CP15_TIMER;
 889	for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 890		arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 891
 892	arch_timer_detect_rate(NULL, np);
 
 
 
 893
 894	arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 895
 896#ifdef CONFIG_FSL_ERRATUM_A008585
 897	if (fsl_a008585_enable < 0)
 898		fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585");
 899	if (fsl_a008585_enable) {
 900		static_branch_enable(&arch_timer_read_ool_enabled);
 901		pr_info("Enabling workaround for FSL erratum A-008585\n");
 902	}
 903#endif
 904
 905	/*
 906	 * If we cannot rely on firmware initializing the timer registers then
 907	 * we should use the physical timers instead.
 908	 */
 909	if (IS_ENABLED(CONFIG_ARM) &&
 910	    of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 911		arch_timer_uses_ppi = PHYS_SECURE_PPI;
 
 
 
 
 
 
 
 912
 913	/* On some systems, the counter stops ticking when in suspend. */
 914	arch_counter_suspend_stop = of_property_read_bool(np,
 915							 "arm,no-tick-in-suspend");
 916
 917	return arch_timer_init();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 918}
 919CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 920CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 921
 922static int __init arch_timer_mem_init(struct device_node *np)
 
 923{
 924	struct device_node *frame, *best_frame = NULL;
 925	void __iomem *cntctlbase, *base;
 926	unsigned int irq, ret = -EINVAL;
 927	u32 cnttidr;
 
 928
 929	arch_timers_present |= ARCH_MEM_TIMER;
 930	cntctlbase = of_iomap(np, 0);
 931	if (!cntctlbase) {
 932		pr_err("arch_timer: Can't find CNTCTLBase\n");
 933		return -ENXIO;
 
 934	}
 935
 936	cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 937
 938	/*
 939	 * Try to find a virtual capable frame. Otherwise fall back to a
 940	 * physical capable frame.
 941	 */
 942	for_each_available_child_of_node(np, frame) {
 943		int n;
 944		u32 cntacr;
 945
 946		if (of_property_read_u32(frame, "frame-number", &n)) {
 947			pr_err("arch_timer: Missing frame-number\n");
 948			of_node_put(frame);
 949			goto out;
 950		}
 951
 952		/* Try enabling everything, and see what sticks */
 953		cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
 954			 CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
 955		writel_relaxed(cntacr, cntctlbase + CNTACR(n));
 956		cntacr = readl_relaxed(cntctlbase + CNTACR(n));
 957
 958		if ((cnttidr & CNTTIDR_VIRT(n)) &&
 959		    !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
 960			of_node_put(best_frame);
 961			best_frame = frame;
 962			arch_timer_mem_use_virtual = true;
 963			break;
 964		}
 965
 966		if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
 967			continue;
 968
 969		of_node_put(best_frame);
 970		best_frame = of_node_get(frame);
 971	}
 972
 973	ret= -ENXIO;
 974	base = arch_counter_base = of_io_request_and_map(best_frame, 0,
 975							 "arch_mem_timer");
 976	if (IS_ERR(base)) {
 977		pr_err("arch_timer: Can't map frame's registers\n");
 978		goto out;
 979	}
 
 
 
 980
 981	if (arch_timer_mem_use_virtual)
 982		irq = irq_of_parse_and_map(best_frame, 1);
 983	else
 984		irq = irq_of_parse_and_map(best_frame, 0);
 985
 986	ret = -EINVAL;
 987	if (!irq) {
 988		pr_err("arch_timer: Frame missing %s irq",
 989		       arch_timer_mem_use_virtual ? "virt" : "phys");
 990		goto out;
 
 
 
 
 
 
 
 
 
 
 991	}
 992
 993	arch_timer_detect_rate(base, np);
 994	ret = arch_timer_mem_register(base, irq);
 995	if (ret)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 996		goto out;
 
 
 997
 998	return arch_timer_common_init();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 999out:
1000	iounmap(cntctlbase);
1001	of_node_put(best_frame);
1002	return ret;
1003}
1004CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1005		       arch_timer_mem_init);
1006
1007#ifdef CONFIG_ACPI
1008static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
 
1009{
1010	int trigger, polarity;
 
 
1011
1012	if (!interrupt)
1013		return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1014
1015	trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
1016			: ACPI_LEVEL_SENSITIVE;
 
1017
1018	polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
1019			: ACPI_ACTIVE_HIGH;
 
 
 
 
 
 
 
 
 
 
 
 
1020
1021	return acpi_register_gsi(NULL, interrupt, trigger, polarity);
 
 
 
 
1022}
1023
1024/* Initialize per-processor generic timer */
1025static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1026{
1027	struct acpi_table_gtdt *gtdt;
1028
1029	if (arch_timers_present & ARCH_CP15_TIMER) {
1030		pr_warn("arch_timer: already initialized, skipping\n");
1031		return -EINVAL;
1032	}
1033
1034	gtdt = container_of(table, struct acpi_table_gtdt, header);
1035
1036	arch_timers_present |= ARCH_CP15_TIMER;
 
 
 
 
1037
1038	arch_timer_ppi[PHYS_SECURE_PPI] =
1039		map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
1040		gtdt->secure_el1_flags);
1041
1042	arch_timer_ppi[PHYS_NONSECURE_PPI] =
1043		map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
1044		gtdt->non_secure_el1_flags);
1045
1046	arch_timer_ppi[VIRT_PPI] =
1047		map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
1048		gtdt->virtual_timer_flags);
1049
1050	arch_timer_ppi[HYP_PPI] =
1051		map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
1052		gtdt->non_secure_el2_flags);
1053
1054	/* Get the frequency from CNTFRQ */
1055	arch_timer_detect_rate(NULL, NULL);
 
 
 
 
 
 
 
 
 
 
 
 
 
1056
1057	/* Always-on capability */
1058	arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
1059
1060	arch_timer_init();
1061	return 0;
 
 
 
 
 
 
 
 
 
 
1062}
1063CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1064#endif