1/*
   2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3 *
   4 * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5 * 2010 (c) MontaVista Software, LLC.
   6 *
   7 * Copied from ARMv6 code, with the low level code inspired
   8 *  by the ARMv7 Oprofile code.
   9 *
  10 * Cortex-A8 has up to 4 configurable performance counters and
  11 *  a single cycle counter.
  12 * Cortex-A9 has up to 31 configurable performance counters and
  13 *  a single cycle counter.
  14 *
  15 * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16 *  counter and all 4 performance counters together can be reset separately.
  17 */
  18
  19#ifdef CONFIG_CPU_V7
  20
  21#include <asm/cp15.h>
  22#include <asm/cputype.h>
  23#include <asm/irq_regs.h>
  24#include <asm/vfp.h>
  25#include "../vfp/vfpinstr.h"
  26
  27#include <linux/of.h>
  28#include <linux/perf/arm_pmu.h>
  29#include <linux/platform_device.h>
  30
  31/*
  32 * Common ARMv7 event types
  33 *
  34 * Note: An implementation may not be able to count all of these events
  35 * but the encodings are considered to be `reserved' in the case that
  36 * they are not available.
  37 */
  38#define ARMV7_PERFCTR_PMNC_SW_INCR			0x00
  39#define ARMV7_PERFCTR_L1_ICACHE_REFILL			0x01
  40#define ARMV7_PERFCTR_ITLB_REFILL			0x02
  41#define ARMV7_PERFCTR_L1_DCACHE_REFILL			0x03
  42#define ARMV7_PERFCTR_L1_DCACHE_ACCESS			0x04
  43#define ARMV7_PERFCTR_DTLB_REFILL			0x05
  44#define ARMV7_PERFCTR_MEM_READ				0x06
  45#define ARMV7_PERFCTR_MEM_WRITE				0x07
  46#define ARMV7_PERFCTR_INSTR_EXECUTED			0x08
  47#define ARMV7_PERFCTR_EXC_TAKEN				0x09
  48#define ARMV7_PERFCTR_EXC_EXECUTED			0x0A
  49#define ARMV7_PERFCTR_CID_WRITE				0x0B
  50
  51/*
  52 * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  53 * It counts:
  54 *  - all (taken) branch instructions,
  55 *  - instructions that explicitly write the PC,
  56 *  - exception generating instructions.
  57 */
  58#define ARMV7_PERFCTR_PC_WRITE				0x0C
  59#define ARMV7_PERFCTR_PC_IMM_BRANCH			0x0D
  60#define ARMV7_PERFCTR_PC_PROC_RETURN			0x0E
  61#define ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS		0x0F
  62#define ARMV7_PERFCTR_PC_BRANCH_MIS_PRED		0x10
  63#define ARMV7_PERFCTR_CLOCK_CYCLES			0x11
  64#define ARMV7_PERFCTR_PC_BRANCH_PRED			0x12
  65
  66/* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
  67#define ARMV7_PERFCTR_MEM_ACCESS			0x13
  68#define ARMV7_PERFCTR_L1_ICACHE_ACCESS			0x14
  69#define ARMV7_PERFCTR_L1_DCACHE_WB			0x15
  70#define ARMV7_PERFCTR_L2_CACHE_ACCESS			0x16
  71#define ARMV7_PERFCTR_L2_CACHE_REFILL			0x17
  72#define ARMV7_PERFCTR_L2_CACHE_WB			0x18
  73#define ARMV7_PERFCTR_BUS_ACCESS			0x19
  74#define ARMV7_PERFCTR_MEM_ERROR				0x1A
  75#define ARMV7_PERFCTR_INSTR_SPEC			0x1B
  76#define ARMV7_PERFCTR_TTBR_WRITE			0x1C
  77#define ARMV7_PERFCTR_BUS_CYCLES			0x1D
  78
  79#define ARMV7_PERFCTR_CPU_CYCLES			0xFF
  80
  81/* ARMv7 Cortex-A8 specific event types */
  82#define ARMV7_A8_PERFCTR_L2_CACHE_ACCESS		0x43
  83#define ARMV7_A8_PERFCTR_L2_CACHE_REFILL		0x44
  84#define ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS		0x50
  85#define ARMV7_A8_PERFCTR_STALL_ISIDE			0x56
  86
  87/* ARMv7 Cortex-A9 specific event types */
  88#define ARMV7_A9_PERFCTR_INSTR_CORE_RENAME		0x68
  89#define ARMV7_A9_PERFCTR_STALL_ICACHE			0x60
  90#define ARMV7_A9_PERFCTR_STALL_DISPATCH			0x66
  91
  92/* ARMv7 Cortex-A5 specific event types */
  93#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL		0xc2
  94#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP		0xc3
  95
  96/* ARMv7 Cortex-A15 specific event types */
  97#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ		0x40
  98#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE	0x41
  99#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ		0x42
 100#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE	0x43
 101
 102#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ		0x4C
 103#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE		0x4D
 104
 105#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ		0x50
 106#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE		0x51
 107#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ		0x52
 108#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE		0x53
 109
 110#define ARMV7_A15_PERFCTR_PC_WRITE_SPEC			0x76
 111
 112/* ARMv7 Cortex-A12 specific event types */
 113#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ		0x40
 114#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE	0x41
 115
 116#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ		0x50
 117#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE		0x51
 118
 119#define ARMV7_A12_PERFCTR_PC_WRITE_SPEC			0x76
 120
 121#define ARMV7_A12_PERFCTR_PF_TLB_REFILL			0xe7
 122
 123/* ARMv7 Krait specific event types */
 124#define KRAIT_PMRESR0_GROUP0				0xcc
 125#define KRAIT_PMRESR1_GROUP0				0xd0
 126#define KRAIT_PMRESR2_GROUP0				0xd4
 127#define KRAIT_VPMRESR0_GROUP0				0xd8
 128
 129#define KRAIT_PERFCTR_L1_ICACHE_ACCESS			0x10011
 130#define KRAIT_PERFCTR_L1_ICACHE_MISS			0x10010
 131
 132#define KRAIT_PERFCTR_L1_ITLB_ACCESS			0x12222
 133#define KRAIT_PERFCTR_L1_DTLB_ACCESS			0x12210
 134
 135/* ARMv7 Scorpion specific event types */
 136#define SCORPION_LPM0_GROUP0				0x4c
 137#define SCORPION_LPM1_GROUP0				0x50
 138#define SCORPION_LPM2_GROUP0				0x54
 139#define SCORPION_L2LPM_GROUP0				0x58
 140#define SCORPION_VLPM_GROUP0				0x5c
 141
 142#define SCORPION_ICACHE_ACCESS				0x10053
 143#define SCORPION_ICACHE_MISS				0x10052
 144
 145#define SCORPION_DTLB_ACCESS				0x12013
 146#define SCORPION_DTLB_MISS				0x12012
 147
 148#define SCORPION_ITLB_MISS				0x12021
 149
 150/*
 151 * Cortex-A8 HW events mapping
 152 *
 153 * The hardware events that we support. We do support cache operations but
 154 * we have harvard caches and no way to combine instruction and data
 155 * accesses/misses in hardware.
 156 */
 157static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 158	PERF_MAP_ALL_UNSUPPORTED,
 159	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
 160	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
 161	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 162	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 163	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_PERFCTR_PC_WRITE,
 164	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 165	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND]	= ARMV7_A8_PERFCTR_STALL_ISIDE,
 166};
 167
 168static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 169					  [PERF_COUNT_HW_CACHE_OP_MAX]
 170					  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 171	PERF_CACHE_MAP_ALL_UNSUPPORTED,
 172
 173	/*
 174	 * The performance counters don't differentiate between read and write
 175	 * accesses/misses so this isn't strictly correct, but it's the best we
 176	 * can do. Writes and reads get combined.
 177	 */
 178	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 179	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 180	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 181	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 182
 183	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 184	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
 185
 186	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 187	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 188	[C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 189	[C(LL)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 190
 191	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 192	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 193
 194	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 195	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 196
 197	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 198	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 199	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 200	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 201};
 202
 203/*
 204 * Cortex-A9 HW events mapping
 205 */
 206static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 207	PERF_MAP_ALL_UNSUPPORTED,
 208	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
 209	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
 210	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 211	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 212	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_PERFCTR_PC_WRITE,
 213	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 214	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND]	= ARMV7_A9_PERFCTR_STALL_ICACHE,
 215	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND]	= ARMV7_A9_PERFCTR_STALL_DISPATCH,
 216};
 217
 218static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 219					  [PERF_COUNT_HW_CACHE_OP_MAX]
 220					  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 221	PERF_CACHE_MAP_ALL_UNSUPPORTED,
 222
 223	/*
 224	 * The performance counters don't differentiate between read and write
 225	 * accesses/misses so this isn't strictly correct, but it's the best we
 226	 * can do. Writes and reads get combined.
 227	 */
 228	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 229	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 230	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 231	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 232
 233	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
 234
 235	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 236	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 237
 238	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 239	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 240
 241	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 242	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 243	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 244	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 245};
 246
 247/*
 248 * Cortex-A5 HW events mapping
 249 */
 250static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
 251	PERF_MAP_ALL_UNSUPPORTED,
 252	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
 253	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
 254	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 255	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 256	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_PERFCTR_PC_WRITE,
 257	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 258};
 259
 260static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 261					[PERF_COUNT_HW_CACHE_OP_MAX]
 262					[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 263	PERF_CACHE_MAP_ALL_UNSUPPORTED,
 264
 265	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 266	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 267	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 268	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 269	[C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)]	= ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 270	[C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)]	= ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 271
 272	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 273	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
 274	/*
 275	 * The prefetch counters don't differentiate between the I side and the
 276	 * D side.
 277	 */
 278	[C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)]	= ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 279	[C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)]	= ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 280
 281	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 282	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 283
 284	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 285	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 286
 287	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 288	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 289	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 290	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 291};
 292
 293/*
 294 * Cortex-A15 HW events mapping
 295 */
 296static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
 297	PERF_MAP_ALL_UNSUPPORTED,
 298	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
 299	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
 300	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 301	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 302	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
 303	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 304	[PERF_COUNT_HW_BUS_CYCLES]		= ARMV7_PERFCTR_BUS_CYCLES,
 305};
 306
 307static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 308					[PERF_COUNT_HW_CACHE_OP_MAX]
 309					[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 310	PERF_CACHE_MAP_ALL_UNSUPPORTED,
 311
 312	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
 313	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
 314	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 315	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
 316
 317	/*
 318	 * Not all performance counters differentiate between read and write
 319	 * accesses/misses so we're not always strictly correct, but it's the
 320	 * best we can do. Writes and reads get combined in these cases.
 321	 */
 322	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 323	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
 324
 325	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
 326	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
 327	[C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
 328	[C(LL)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
 329
 330	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
 331	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
 332
 333	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 334	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 335
 336	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 337	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 338	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 339	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 340};
 341
 342/*
 343 * Cortex-A7 HW events mapping
 344 */
 345static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
 346	PERF_MAP_ALL_UNSUPPORTED,
 347	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
 348	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
 349	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 350	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 351	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_PERFCTR_PC_WRITE,
 352	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 353	[PERF_COUNT_HW_BUS_CYCLES]		= ARMV7_PERFCTR_BUS_CYCLES,
 354};
 355
 356static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 357					[PERF_COUNT_HW_CACHE_OP_MAX]
 358					[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 359	PERF_CACHE_MAP_ALL_UNSUPPORTED,
 360
 361	/*
 362	 * The performance counters don't differentiate between read and write
 363	 * accesses/misses so this isn't strictly correct, but it's the best we
 364	 * can do. Writes and reads get combined.
 365	 */
 366	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 367	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 368	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 369	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 370
 371	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 372	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
 373
 374	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L2_CACHE_ACCESS,
 375	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L2_CACHE_REFILL,
 376	[C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L2_CACHE_ACCESS,
 377	[C(LL)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L2_CACHE_REFILL,
 378
 379	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 380	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 381
 382	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 383	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 384
 385	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 386	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 387	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 388	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 389};
 390
 391/*
 392 * Cortex-A12 HW events mapping
 393 */
 394static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
 395	PERF_MAP_ALL_UNSUPPORTED,
 396	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
 397	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
 398	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 399	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 400	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
 401	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 402	[PERF_COUNT_HW_BUS_CYCLES]		= ARMV7_PERFCTR_BUS_CYCLES,
 403};
 404
 405static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 406					[PERF_COUNT_HW_CACHE_OP_MAX]
 407					[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 408	PERF_CACHE_MAP_ALL_UNSUPPORTED,
 409
 410	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
 411	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 412	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 413	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 414
 415	/*
 416	 * Not all performance counters differentiate between read and write
 417	 * accesses/misses so we're not always strictly correct, but it's the
 418	 * best we can do. Writes and reads get combined in these cases.
 419	 */
 420	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 421	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
 422
 423	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
 424	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L2_CACHE_REFILL,
 425	[C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
 426	[C(LL)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L2_CACHE_REFILL,
 427
 428	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 429	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
 430	[C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)]	= ARMV7_A12_PERFCTR_PF_TLB_REFILL,
 431
 432	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 433	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
 434
 435	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 436	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 437	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 438	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 439};
 440
 441/*
 442 * Krait HW events mapping
 443 */
 444static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
 445	PERF_MAP_ALL_UNSUPPORTED,
 446	[PERF_COUNT_HW_CPU_CYCLES]	    = ARMV7_PERFCTR_CPU_CYCLES,
 447	[PERF_COUNT_HW_INSTRUCTIONS]	    = ARMV7_PERFCTR_INSTR_EXECUTED,
 448	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 449	[PERF_COUNT_HW_BRANCH_MISSES]	    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 450	[PERF_COUNT_HW_BUS_CYCLES]	    = ARMV7_PERFCTR_CLOCK_CYCLES,
 451};
 452
 453static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
 454	PERF_MAP_ALL_UNSUPPORTED,
 455	[PERF_COUNT_HW_CPU_CYCLES]	    = ARMV7_PERFCTR_CPU_CYCLES,
 456	[PERF_COUNT_HW_INSTRUCTIONS]	    = ARMV7_PERFCTR_INSTR_EXECUTED,
 457	[PERF_COUNT_HW_BRANCH_MISSES]	    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 458	[PERF_COUNT_HW_BUS_CYCLES]	    = ARMV7_PERFCTR_CLOCK_CYCLES,
 459};
 460
 461static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 462					  [PERF_COUNT_HW_CACHE_OP_MAX]
 463					  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 464	PERF_CACHE_MAP_ALL_UNSUPPORTED,
 465
 466	/*
 467	 * The performance counters don't differentiate between read and write
 468	 * accesses/misses so this isn't strictly correct, but it's the best we
 469	 * can do. Writes and reads get combined.
 470	 */
 471	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 472	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 473	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 474	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
 475
 476	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_ICACHE_ACCESS,
 477	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= KRAIT_PERFCTR_L1_ICACHE_MISS,
 478
 479	[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_DTLB_ACCESS,
 480	[C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_DTLB_ACCESS,
 481
 482	[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_ITLB_ACCESS,
 483	[C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_ITLB_ACCESS,
 484
 485	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 486	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 487	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
 488	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 489};
 490
 491/*
 492 * Scorpion HW events mapping
 493 */
 494static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
 495	PERF_MAP_ALL_UNSUPPORTED,
 496	[PERF_COUNT_HW_CPU_CYCLES]	    = ARMV7_PERFCTR_CPU_CYCLES,
 497	[PERF_COUNT_HW_INSTRUCTIONS]	    = ARMV7_PERFCTR_INSTR_EXECUTED,
 498	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 499	[PERF_COUNT_HW_BRANCH_MISSES]	    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 500	[PERF_COUNT_HW_BUS_CYCLES]	    = ARMV7_PERFCTR_CLOCK_CYCLES,
 501};
 502
 503static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 504					    [PERF_COUNT_HW_CACHE_OP_MAX]
 505					    [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 506	PERF_CACHE_MAP_ALL_UNSUPPORTED,
 507	/*
 508	 * The performance counters don't differentiate between read and write
 509	 * accesses/misses so this isn't strictly correct, but it's the best we
 510	 * can do. Writes and reads get combined.
 511	 */
 512	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 513	[C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 514	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 515	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 516	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
 517	[C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
 518	/*
 519	 * Only ITLB misses and DTLB refills are supported.  If users want the
 520	 * DTLB refills misses a raw counter must be used.
 521	 */
 522	[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 523	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 524	[C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 525	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 526	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 527	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 528	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 529	[C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 530	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 531	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 532};
 533
 534PMU_FORMAT_ATTR(event, "config:0-7");
 535
 536static struct attribute *armv7_pmu_format_attrs[] = {
 537	&format_attr_event.attr,
 538	NULL,
 539};
 540
 541static struct attribute_group armv7_pmu_format_attr_group = {
 542	.name = "format",
 543	.attrs = armv7_pmu_format_attrs,
 544};
 545
 546#define ARMV7_EVENT_ATTR_RESOLVE(m) #m
 547#define ARMV7_EVENT_ATTR(name, config) \
 548	PMU_EVENT_ATTR_STRING(name, armv7_event_attr_##name, \
 549			      "event=" ARMV7_EVENT_ATTR_RESOLVE(config))
 550
 551ARMV7_EVENT_ATTR(sw_incr, ARMV7_PERFCTR_PMNC_SW_INCR);
 552ARMV7_EVENT_ATTR(l1i_cache_refill, ARMV7_PERFCTR_L1_ICACHE_REFILL);
 553ARMV7_EVENT_ATTR(l1i_tlb_refill, ARMV7_PERFCTR_ITLB_REFILL);
 554ARMV7_EVENT_ATTR(l1d_cache_refill, ARMV7_PERFCTR_L1_DCACHE_REFILL);
 555ARMV7_EVENT_ATTR(l1d_cache, ARMV7_PERFCTR_L1_DCACHE_ACCESS);
 556ARMV7_EVENT_ATTR(l1d_tlb_refill, ARMV7_PERFCTR_DTLB_REFILL);
 557ARMV7_EVENT_ATTR(ld_retired, ARMV7_PERFCTR_MEM_READ);
 558ARMV7_EVENT_ATTR(st_retired, ARMV7_PERFCTR_MEM_WRITE);
 559ARMV7_EVENT_ATTR(inst_retired, ARMV7_PERFCTR_INSTR_EXECUTED);
 560ARMV7_EVENT_ATTR(exc_taken, ARMV7_PERFCTR_EXC_TAKEN);
 561ARMV7_EVENT_ATTR(exc_return, ARMV7_PERFCTR_EXC_EXECUTED);
 562ARMV7_EVENT_ATTR(cid_write_retired, ARMV7_PERFCTR_CID_WRITE);
 563ARMV7_EVENT_ATTR(pc_write_retired, ARMV7_PERFCTR_PC_WRITE);
 564ARMV7_EVENT_ATTR(br_immed_retired, ARMV7_PERFCTR_PC_IMM_BRANCH);
 565ARMV7_EVENT_ATTR(br_return_retired, ARMV7_PERFCTR_PC_PROC_RETURN);
 566ARMV7_EVENT_ATTR(unaligned_ldst_retired, ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS);
 567ARMV7_EVENT_ATTR(br_mis_pred, ARMV7_PERFCTR_PC_BRANCH_MIS_PRED);
 568ARMV7_EVENT_ATTR(cpu_cycles, ARMV7_PERFCTR_CLOCK_CYCLES);
 569ARMV7_EVENT_ATTR(br_pred, ARMV7_PERFCTR_PC_BRANCH_PRED);
 570
 571static struct attribute *armv7_pmuv1_event_attrs[] = {
 572	&armv7_event_attr_sw_incr.attr.attr,
 573	&armv7_event_attr_l1i_cache_refill.attr.attr,
 574	&armv7_event_attr_l1i_tlb_refill.attr.attr,
 575	&armv7_event_attr_l1d_cache_refill.attr.attr,
 576	&armv7_event_attr_l1d_cache.attr.attr,
 577	&armv7_event_attr_l1d_tlb_refill.attr.attr,
 578	&armv7_event_attr_ld_retired.attr.attr,
 579	&armv7_event_attr_st_retired.attr.attr,
 580	&armv7_event_attr_inst_retired.attr.attr,
 581	&armv7_event_attr_exc_taken.attr.attr,
 582	&armv7_event_attr_exc_return.attr.attr,
 583	&armv7_event_attr_cid_write_retired.attr.attr,
 584	&armv7_event_attr_pc_write_retired.attr.attr,
 585	&armv7_event_attr_br_immed_retired.attr.attr,
 586	&armv7_event_attr_br_return_retired.attr.attr,
 587	&armv7_event_attr_unaligned_ldst_retired.attr.attr,
 588	&armv7_event_attr_br_mis_pred.attr.attr,
 589	&armv7_event_attr_cpu_cycles.attr.attr,
 590	&armv7_event_attr_br_pred.attr.attr,
 591	NULL,
 592};
 593
 594static struct attribute_group armv7_pmuv1_events_attr_group = {
 595	.name = "events",
 596	.attrs = armv7_pmuv1_event_attrs,
 597};
 598
 599static const struct attribute_group *armv7_pmuv1_attr_groups[] = {
 600	&armv7_pmuv1_events_attr_group,
 601	&armv7_pmu_format_attr_group,
 602	NULL,
 603};
 604
 605ARMV7_EVENT_ATTR(mem_access, ARMV7_PERFCTR_MEM_ACCESS);
 606ARMV7_EVENT_ATTR(l1i_cache, ARMV7_PERFCTR_L1_ICACHE_ACCESS);
 607ARMV7_EVENT_ATTR(l1d_cache_wb, ARMV7_PERFCTR_L1_DCACHE_WB);
 608ARMV7_EVENT_ATTR(l2d_cache, ARMV7_PERFCTR_L2_CACHE_ACCESS);
 609ARMV7_EVENT_ATTR(l2d_cache_refill, ARMV7_PERFCTR_L2_CACHE_REFILL);
 610ARMV7_EVENT_ATTR(l2d_cache_wb, ARMV7_PERFCTR_L2_CACHE_WB);
 611ARMV7_EVENT_ATTR(bus_access, ARMV7_PERFCTR_BUS_ACCESS);
 612ARMV7_EVENT_ATTR(memory_error, ARMV7_PERFCTR_MEM_ERROR);
 613ARMV7_EVENT_ATTR(inst_spec, ARMV7_PERFCTR_INSTR_SPEC);
 614ARMV7_EVENT_ATTR(ttbr_write_retired, ARMV7_PERFCTR_TTBR_WRITE);
 615ARMV7_EVENT_ATTR(bus_cycles, ARMV7_PERFCTR_BUS_CYCLES);
 616
 617static struct attribute *armv7_pmuv2_event_attrs[] = {
 618	&armv7_event_attr_sw_incr.attr.attr,
 619	&armv7_event_attr_l1i_cache_refill.attr.attr,
 620	&armv7_event_attr_l1i_tlb_refill.attr.attr,
 621	&armv7_event_attr_l1d_cache_refill.attr.attr,
 622	&armv7_event_attr_l1d_cache.attr.attr,
 623	&armv7_event_attr_l1d_tlb_refill.attr.attr,
 624	&armv7_event_attr_ld_retired.attr.attr,
 625	&armv7_event_attr_st_retired.attr.attr,
 626	&armv7_event_attr_inst_retired.attr.attr,
 627	&armv7_event_attr_exc_taken.attr.attr,
 628	&armv7_event_attr_exc_return.attr.attr,
 629	&armv7_event_attr_cid_write_retired.attr.attr,
 630	&armv7_event_attr_pc_write_retired.attr.attr,
 631	&armv7_event_attr_br_immed_retired.attr.attr,
 632	&armv7_event_attr_br_return_retired.attr.attr,
 633	&armv7_event_attr_unaligned_ldst_retired.attr.attr,
 634	&armv7_event_attr_br_mis_pred.attr.attr,
 635	&armv7_event_attr_cpu_cycles.attr.attr,
 636	&armv7_event_attr_br_pred.attr.attr,
 637	&armv7_event_attr_mem_access.attr.attr,
 638	&armv7_event_attr_l1i_cache.attr.attr,
 639	&armv7_event_attr_l1d_cache_wb.attr.attr,
 640	&armv7_event_attr_l2d_cache.attr.attr,
 641	&armv7_event_attr_l2d_cache_refill.attr.attr,
 642	&armv7_event_attr_l2d_cache_wb.attr.attr,
 643	&armv7_event_attr_bus_access.attr.attr,
 644	&armv7_event_attr_memory_error.attr.attr,
 645	&armv7_event_attr_inst_spec.attr.attr,
 646	&armv7_event_attr_ttbr_write_retired.attr.attr,
 647	&armv7_event_attr_bus_cycles.attr.attr,
 648	NULL,
 649};
 650
 651static struct attribute_group armv7_pmuv2_events_attr_group = {
 652	.name = "events",
 653	.attrs = armv7_pmuv2_event_attrs,
 654};
 655
 656static const struct attribute_group *armv7_pmuv2_attr_groups[] = {
 657	&armv7_pmuv2_events_attr_group,
 658	&armv7_pmu_format_attr_group,
 659	NULL,
 660};
 661
 662/*
 663 * Perf Events' indices
 664 */
 665#define	ARMV7_IDX_CYCLE_COUNTER	0
 666#define	ARMV7_IDX_COUNTER0	1
 667#define	ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
 668	(ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 669
 670#define	ARMV7_MAX_COUNTERS	32
 671#define	ARMV7_COUNTER_MASK	(ARMV7_MAX_COUNTERS - 1)
 672
 673/*
 674 * ARMv7 low level PMNC access
 675 */
 676
 677/*
 678 * Perf Event to low level counters mapping
 679 */
 680#define	ARMV7_IDX_TO_COUNTER(x)	\
 681	(((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
 682
 683/*
 684 * Per-CPU PMNC: config reg
 685 */
 686#define ARMV7_PMNC_E		(1 << 0) /* Enable all counters */
 687#define ARMV7_PMNC_P		(1 << 1) /* Reset all counters */
 688#define ARMV7_PMNC_C		(1 << 2) /* Cycle counter reset */
 689#define ARMV7_PMNC_D		(1 << 3) /* CCNT counts every 64th cpu cycle */
 690#define ARMV7_PMNC_X		(1 << 4) /* Export to ETM */
 691#define ARMV7_PMNC_DP		(1 << 5) /* Disable CCNT if non-invasive debug*/
 692#define	ARMV7_PMNC_N_SHIFT	11	 /* Number of counters supported */
 693#define	ARMV7_PMNC_N_MASK	0x1f
 694#define	ARMV7_PMNC_MASK		0x3f	 /* Mask for writable bits */
 695
 696/*
 697 * FLAG: counters overflow flag status reg
 698 */
 699#define	ARMV7_FLAG_MASK		0xffffffff	/* Mask for writable bits */
 700#define	ARMV7_OVERFLOWED_MASK	ARMV7_FLAG_MASK
 701
 702/*
 703 * PMXEVTYPER: Event selection reg
 704 */
 705#define	ARMV7_EVTYPE_MASK	0xc80000ff	/* Mask for writable bits */
 706#define	ARMV7_EVTYPE_EVENT	0xff		/* Mask for EVENT bits */
 707
 708/*
 709 * Event filters for PMUv2
 710 */
 711#define	ARMV7_EXCLUDE_PL1	(1 << 31)
 712#define	ARMV7_EXCLUDE_USER	(1 << 30)
 713#define	ARMV7_INCLUDE_HYP	(1 << 27)
 714
 715/*
 716 * Secure debug enable reg
 717 */
 718#define ARMV7_SDER_SUNIDEN	BIT(1) /* Permit non-invasive debug */
 719
 720static inline u32 armv7_pmnc_read(void)
 721{
 722	u32 val;
 723	asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 724	return val;
 725}
 726
 727static inline void armv7_pmnc_write(u32 val)
 728{
 729	val &= ARMV7_PMNC_MASK;
 730	isb();
 731	asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 732}
 733
 734static inline int armv7_pmnc_has_overflowed(u32 pmnc)
 735{
 736	return pmnc & ARMV7_OVERFLOWED_MASK;
 737}
 738
 739static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
 740{
 741	return idx >= ARMV7_IDX_CYCLE_COUNTER &&
 742		idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
 743}
 744
 745static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
 746{
 747	return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
 748}
 749
 750static inline void armv7_pmnc_select_counter(int idx)
 751{
 752	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 753	asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
 754	isb();
 755}
 756
 757static inline u32 armv7pmu_read_counter(struct perf_event *event)
 758{
 759	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 760	struct hw_perf_event *hwc = &event->hw;
 761	int idx = hwc->idx;
 762	u32 value = 0;
 763
 764	if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 765		pr_err("CPU%u reading wrong counter %d\n",
 766			smp_processor_id(), idx);
 767	} else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 768		asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 769	} else {
 770		armv7_pmnc_select_counter(idx);
 771		asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
 772	}
 773
 774	return value;
 775}
 776
 777static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
 778{
 779	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 780	struct hw_perf_event *hwc = &event->hw;
 781	int idx = hwc->idx;
 782
 783	if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 784		pr_err("CPU%u writing wrong counter %d\n",
 785			smp_processor_id(), idx);
 786	} else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 787		asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 788	} else {
 789		armv7_pmnc_select_counter(idx);
 790		asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
 791	}
 792}
 793
 794static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
 795{
 796	armv7_pmnc_select_counter(idx);
 797	val &= ARMV7_EVTYPE_MASK;
 798	asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 799}
 800
 801static inline void armv7_pmnc_enable_counter(int idx)
 802{
 803	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 804	asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
 805}
 806
 807static inline void armv7_pmnc_disable_counter(int idx)
 808{
 809	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 810	asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
 811}
 812
 813static inline void armv7_pmnc_enable_intens(int idx)
 814{
 815	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 816	asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
 817}
 818
 819static inline void armv7_pmnc_disable_intens(int idx)
 820{
 821	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 822	asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
 823	isb();
 824	/* Clear the overflow flag in case an interrupt is pending. */
 825	asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
 826	isb();
 827}
 828
 829static inline u32 armv7_pmnc_getreset_flags(void)
 830{
 831	u32 val;
 832
 833	/* Read */
 834	asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 835
 836	/* Write to clear flags */
 837	val &= ARMV7_FLAG_MASK;
 838	asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 839
 840	return val;
 841}
 842
 843#ifdef DEBUG
 844static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
 845{
 846	u32 val;
 847	unsigned int cnt;
 848
 849	pr_info("PMNC registers dump:\n");
 850
 851	asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 852	pr_info("PMNC  =0x%08x\n", val);
 853
 854	asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 855	pr_info("CNTENS=0x%08x\n", val);
 856
 857	asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 858	pr_info("INTENS=0x%08x\n", val);
 859
 860	asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 861	pr_info("FLAGS =0x%08x\n", val);
 862
 863	asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 864	pr_info("SELECT=0x%08x\n", val);
 865
 866	asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 867	pr_info("CCNT  =0x%08x\n", val);
 868
 869	for (cnt = ARMV7_IDX_COUNTER0;
 870			cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
 871		armv7_pmnc_select_counter(cnt);
 872		asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 873		pr_info("CNT[%d] count =0x%08x\n",
 874			ARMV7_IDX_TO_COUNTER(cnt), val);
 875		asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 876		pr_info("CNT[%d] evtsel=0x%08x\n",
 877			ARMV7_IDX_TO_COUNTER(cnt), val);
 878	}
 879}
 880#endif
 881
 882static void armv7pmu_enable_event(struct perf_event *event)
 883{
 884	unsigned long flags;
 885	struct hw_perf_event *hwc = &event->hw;
 886	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 887	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 888	int idx = hwc->idx;
 889
 890	if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 891		pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 892			smp_processor_id(), idx);
 893		return;
 894	}
 895
 896	/*
 897	 * Enable counter and interrupt, and set the counter to count
 898	 * the event that we're interested in.
 899	 */
 900	raw_spin_lock_irqsave(&events->pmu_lock, flags);
 901
 902	/*
 903	 * Disable counter
 904	 */
 905	armv7_pmnc_disable_counter(idx);
 906
 907	/*
 908	 * Set event (if destined for PMNx counters)
 909	 * We only need to set the event for the cycle counter if we
 910	 * have the ability to perform event filtering.
 911	 */
 912	if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
 913		armv7_pmnc_write_evtsel(idx, hwc->config_base);
 914
 915	/*
 916	 * Enable interrupt for this counter
 917	 */
 918	armv7_pmnc_enable_intens(idx);
 919
 920	/*
 921	 * Enable counter
 922	 */
 923	armv7_pmnc_enable_counter(idx);
 924
 925	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 926}
 927
 928static void armv7pmu_disable_event(struct perf_event *event)
 929{
 930	unsigned long flags;
 931	struct hw_perf_event *hwc = &event->hw;
 932	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 933	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 934	int idx = hwc->idx;
 935
 936	if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 937		pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 938			smp_processor_id(), idx);
 939		return;
 940	}
 941
 942	/*
 943	 * Disable counter and interrupt
 944	 */
 945	raw_spin_lock_irqsave(&events->pmu_lock, flags);
 946
 947	/*
 948	 * Disable counter
 949	 */
 950	armv7_pmnc_disable_counter(idx);
 951
 952	/*
 953	 * Disable interrupt for this counter
 954	 */
 955	armv7_pmnc_disable_intens(idx);
 956
 957	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 958}
 959
 960static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
 961{
 962	u32 pmnc;
 963	struct perf_sample_data data;
 964	struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 965	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 966	struct pt_regs *regs;
 967	int idx;
 968
 969	/*
 970	 * Get and reset the IRQ flags
 971	 */
 972	pmnc = armv7_pmnc_getreset_flags();
 973
 974	/*
 975	 * Did an overflow occur?
 976	 */
 977	if (!armv7_pmnc_has_overflowed(pmnc))
 978		return IRQ_NONE;
 979
 980	/*
 981	 * Handle the counter(s) overflow(s)
 982	 */
 983	regs = get_irq_regs();
 984
 985	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 986		struct perf_event *event = cpuc->events[idx];
 987		struct hw_perf_event *hwc;
 988
 989		/* Ignore if we don't have an event. */
 990		if (!event)
 991			continue;
 992
 993		/*
 994		 * We have a single interrupt for all counters. Check that
 995		 * each counter has overflowed before we process it.
 996		 */
 997		if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
 998			continue;
 999
1000		hwc = &event->hw;
1001		armpmu_event_update(event);
1002		perf_sample_data_init(&data, 0, hwc->last_period);
1003		if (!armpmu_event_set_period(event))
1004			continue;
1005
1006		if (perf_event_overflow(event, &data, regs))
1007			cpu_pmu->disable(event);
1008	}
1009
1010	/*
1011	 * Handle the pending perf events.
1012	 *
1013	 * Note: this call *must* be run with interrupts disabled. For
1014	 * platforms that can have the PMU interrupts raised as an NMI, this
1015	 * will not work.
1016	 */
1017	irq_work_run();
1018
1019	return IRQ_HANDLED;
1020}
1021
1022static void armv7pmu_start(struct arm_pmu *cpu_pmu)
1023{
1024	unsigned long flags;
1025	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1026
1027	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1028	/* Enable all counters */
1029	armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
1030	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1031}
1032
1033static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
1034{
1035	unsigned long flags;
1036	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1037
1038	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1039	/* Disable all counters */
1040	armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
1041	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1042}
1043
1044static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
1045				  struct perf_event *event)
1046{
1047	int idx;
1048	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1049	struct hw_perf_event *hwc = &event->hw;
1050	unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
1051
1052	/* Always place a cycle counter into the cycle counter. */
1053	if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
1054		if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
1055			return -EAGAIN;
1056
1057		return ARMV7_IDX_CYCLE_COUNTER;
1058	}
1059
1060	/*
1061	 * For anything other than a cycle counter, try and use
1062	 * the events counters
1063	 */
1064	for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1065		if (!test_and_set_bit(idx, cpuc->used_mask))
1066			return idx;
1067	}
1068
1069	/* The counters are all in use. */
1070	return -EAGAIN;
1071}
1072
1073/*
1074 * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1075 */
1076static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1077				     struct perf_event_attr *attr)
1078{
1079	unsigned long config_base = 0;
1080
1081	if (attr->exclude_idle)
1082		return -EPERM;
1083	if (attr->exclude_user)
1084		config_base |= ARMV7_EXCLUDE_USER;
1085	if (attr->exclude_kernel)
1086		config_base |= ARMV7_EXCLUDE_PL1;
1087	if (!attr->exclude_hv)
1088		config_base |= ARMV7_INCLUDE_HYP;
1089
1090	/*
1091	 * Install the filter into config_base as this is used to
1092	 * construct the event type.
1093	 */
1094	event->config_base = config_base;
1095
1096	return 0;
1097}
1098
1099static void armv7pmu_reset(void *info)
1100{
1101	struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
1102	u32 idx, nb_cnt = cpu_pmu->num_events, val;
1103
1104	if (cpu_pmu->secure_access) {
1105		asm volatile("mrc p15, 0, %0, c1, c1, 1" : "=r" (val));
1106		val |= ARMV7_SDER_SUNIDEN;
1107		asm volatile("mcr p15, 0, %0, c1, c1, 1" : : "r" (val));
1108	}
1109
1110	/* The counter and interrupt enable registers are unknown at reset. */
1111	for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1112		armv7_pmnc_disable_counter(idx);
1113		armv7_pmnc_disable_intens(idx);
1114	}
1115
1116	/* Initialize & Reset PMNC: C and P bits */
1117	armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1118}
1119
1120static int armv7_a8_map_event(struct perf_event *event)
1121{
1122	return armpmu_map_event(event, &armv7_a8_perf_map,
1123				&armv7_a8_perf_cache_map, 0xFF);
1124}
1125
1126static int armv7_a9_map_event(struct perf_event *event)
1127{
1128	return armpmu_map_event(event, &armv7_a9_perf_map,
1129				&armv7_a9_perf_cache_map, 0xFF);
1130}
1131
1132static int armv7_a5_map_event(struct perf_event *event)
1133{
1134	return armpmu_map_event(event, &armv7_a5_perf_map,
1135				&armv7_a5_perf_cache_map, 0xFF);
1136}
1137
1138static int armv7_a15_map_event(struct perf_event *event)
1139{
1140	return armpmu_map_event(event, &armv7_a15_perf_map,
1141				&armv7_a15_perf_cache_map, 0xFF);
1142}
1143
1144static int armv7_a7_map_event(struct perf_event *event)
1145{
1146	return armpmu_map_event(event, &armv7_a7_perf_map,
1147				&armv7_a7_perf_cache_map, 0xFF);
1148}
1149
1150static int armv7_a12_map_event(struct perf_event *event)
1151{
1152	return armpmu_map_event(event, &armv7_a12_perf_map,
1153				&armv7_a12_perf_cache_map, 0xFF);
1154}
1155
1156static int krait_map_event(struct perf_event *event)
1157{
1158	return armpmu_map_event(event, &krait_perf_map,
1159				&krait_perf_cache_map, 0xFFFFF);
1160}
1161
1162static int krait_map_event_no_branch(struct perf_event *event)
1163{
1164	return armpmu_map_event(event, &krait_perf_map_no_branch,
1165				&krait_perf_cache_map, 0xFFFFF);
1166}
1167
1168static int scorpion_map_event(struct perf_event *event)
1169{
1170	return armpmu_map_event(event, &scorpion_perf_map,
1171				&scorpion_perf_cache_map, 0xFFFFF);
1172}
1173
1174static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1175{
1176	cpu_pmu->handle_irq	= armv7pmu_handle_irq;
1177	cpu_pmu->enable		= armv7pmu_enable_event;
1178	cpu_pmu->disable	= armv7pmu_disable_event;
1179	cpu_pmu->read_counter	= armv7pmu_read_counter;
1180	cpu_pmu->write_counter	= armv7pmu_write_counter;
1181	cpu_pmu->get_event_idx	= armv7pmu_get_event_idx;
1182	cpu_pmu->start		= armv7pmu_start;
1183	cpu_pmu->stop		= armv7pmu_stop;
1184	cpu_pmu->reset		= armv7pmu_reset;
1185	cpu_pmu->max_period	= (1LLU << 32) - 1;
1186};
1187
1188static void armv7_read_num_pmnc_events(void *info)
1189{
1190	int *nb_cnt = info;
1191
1192	/* Read the nb of CNTx counters supported from PMNC */
1193	*nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1194
1195	/* Add the CPU cycles counter */
1196	*nb_cnt += 1;
1197}
1198
1199static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
1200{
1201	return smp_call_function_any(&arm_pmu->supported_cpus,
1202				     armv7_read_num_pmnc_events,
1203				     &arm_pmu->num_events, 1);
1204}
1205
1206static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1207{
1208	armv7pmu_init(cpu_pmu);
1209	cpu_pmu->name		= "armv7_cortex_a8";
1210	cpu_pmu->map_event	= armv7_a8_map_event;
1211	cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1212	return armv7_probe_num_events(cpu_pmu);
1213}
1214
1215static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1216{
1217	armv7pmu_init(cpu_pmu);
1218	cpu_pmu->name		= "armv7_cortex_a9";
1219	cpu_pmu->map_event	= armv7_a9_map_event;
1220	cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1221	return armv7_probe_num_events(cpu_pmu);
1222}
1223
1224static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1225{
1226	armv7pmu_init(cpu_pmu);
1227	cpu_pmu->name		= "armv7_cortex_a5";
1228	cpu_pmu->map_event	= armv7_a5_map_event;
1229	cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1230	return armv7_probe_num_events(cpu_pmu);
1231}
1232
1233static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1234{
1235	armv7pmu_init(cpu_pmu);
1236	cpu_pmu->name		= "armv7_cortex_a15";
1237	cpu_pmu->map_event	= armv7_a15_map_event;
1238	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1239	cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1240	return armv7_probe_num_events(cpu_pmu);
1241}
1242
1243static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1244{
1245	armv7pmu_init(cpu_pmu);
1246	cpu_pmu->name		= "armv7_cortex_a7";
1247	cpu_pmu->map_event	= armv7_a7_map_event;
1248	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1249	cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1250	return armv7_probe_num_events(cpu_pmu);
1251}
1252
1253static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1254{
1255	armv7pmu_init(cpu_pmu);
1256	cpu_pmu->name		= "armv7_cortex_a12";
1257	cpu_pmu->map_event	= armv7_a12_map_event;
1258	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1259	cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1260	return armv7_probe_num_events(cpu_pmu);
1261}
1262
1263static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1264{
1265	int ret = armv7_a12_pmu_init(cpu_pmu);
1266	cpu_pmu->name = "armv7_cortex_a17";
1267	cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1268	return ret;
1269}
1270
1271/*
1272 * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1273 *
1274 *            31   30     24     16     8      0
1275 *            +--------------------------------+
1276 *  PMRESR0   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1277 *            +--------------------------------+
1278 *  PMRESR1   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1279 *            +--------------------------------+
1280 *  PMRESR2   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1281 *            +--------------------------------+
1282 *  VPMRESR0  | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1283 *            +--------------------------------+
1284 *              EN | G=3  | G=2  | G=1  | G=0
1285 *
1286 *  Event Encoding:
1287 *
1288 *      hwc->config_base = 0xNRCCG
1289 *
1290 *      N  = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1291 *      R  = region register
1292 *      CC = class of events the group G is choosing from
1293 *      G  = group or particular event
1294 *
1295 *  Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1296 *
1297 *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1298 *  unit, etc.) while the event code (CC) corresponds to a particular class of
1299 *  events (interrupts for example). An event code is broken down into
1300 *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1301 *  example).
1302 */
1303
1304#define KRAIT_EVENT		(1 << 16)
1305#define VENUM_EVENT		(2 << 16)
1306#define KRAIT_EVENT_MASK	(KRAIT_EVENT | VENUM_EVENT)
1307#define PMRESRn_EN		BIT(31)
1308
1309#define EVENT_REGION(event)	(((event) >> 12) & 0xf)		/* R */
1310#define EVENT_GROUP(event)	((event) & 0xf)			/* G */
1311#define EVENT_CODE(event)	(((event) >> 4) & 0xff)		/* CC */
1312#define EVENT_VENUM(event)	(!!(event & VENUM_EVENT))	/* N=2 */
1313#define EVENT_CPU(event)	(!!(event & KRAIT_EVENT))	/* N=1 */
1314
1315static u32 krait_read_pmresrn(int n)
1316{
1317	u32 val;
1318
1319	switch (n) {
1320	case 0:
1321		asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1322		break;
1323	case 1:
1324		asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1325		break;
1326	case 2:
1327		asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1328		break;
1329	default:
1330		BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1331	}
1332
1333	return val;
1334}
1335
1336static void krait_write_pmresrn(int n, u32 val)
1337{
1338	switch (n) {
1339	case 0:
1340		asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1341		break;
1342	case 1:
1343		asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1344		break;
1345	case 2:
1346		asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1347		break;
1348	default:
1349		BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1350	}
1351}
1352
1353static u32 venum_read_pmresr(void)
1354{
1355	u32 val;
1356	asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1357	return val;
1358}
1359
1360static void venum_write_pmresr(u32 val)
1361{
1362	asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1363}
1364
1365static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1366{
1367	u32 venum_new_val;
1368	u32 fp_new_val;
1369
1370	BUG_ON(preemptible());
1371	/* CPACR Enable CP10 and CP11 access */
1372	*venum_orig_val = get_copro_access();
1373	venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1374	set_copro_access(venum_new_val);
1375
1376	/* Enable FPEXC */
1377	*fp_orig_val = fmrx(FPEXC);
1378	fp_new_val = *fp_orig_val | FPEXC_EN;
1379	fmxr(FPEXC, fp_new_val);
1380}
1381
1382static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1383{
1384	BUG_ON(preemptible());
1385	/* Restore FPEXC */
1386	fmxr(FPEXC, fp_orig_val);
1387	isb();
1388	/* Restore CPACR */
1389	set_copro_access(venum_orig_val);
1390}
1391
1392static u32 krait_get_pmresrn_event(unsigned int region)
1393{
1394	static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1395					     KRAIT_PMRESR1_GROUP0,
1396					     KRAIT_PMRESR2_GROUP0 };
1397	return pmresrn_table[region];
1398}
1399
1400static void krait_evt_setup(int idx, u32 config_base)
1401{
1402	u32 val;
1403	u32 mask;
1404	u32 vval, fval;
1405	unsigned int region = EVENT_REGION(config_base);
1406	unsigned int group = EVENT_GROUP(config_base);
1407	unsigned int code = EVENT_CODE(config_base);
1408	unsigned int group_shift;
1409	bool venum_event = EVENT_VENUM(config_base);
1410
1411	group_shift = group * 8;
1412	mask = 0xff << group_shift;
1413
1414	/* Configure evtsel for the region and group */
1415	if (venum_event)
1416		val = KRAIT_VPMRESR0_GROUP0;
1417	else
1418		val = krait_get_pmresrn_event(region);
1419	val += group;
1420	/* Mix in mode-exclusion bits */
1421	val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1422	armv7_pmnc_write_evtsel(idx, val);
1423
1424	if (venum_event) {
1425		venum_pre_pmresr(&vval, &fval);
1426		val = venum_read_pmresr();
1427		val &= ~mask;
1428		val |= code << group_shift;
1429		val |= PMRESRn_EN;
1430		venum_write_pmresr(val);
1431		venum_post_pmresr(vval, fval);
1432	} else {
1433		val = krait_read_pmresrn(region);
1434		val &= ~mask;
1435		val |= code << group_shift;
1436		val |= PMRESRn_EN;
1437		krait_write_pmresrn(region, val);
1438	}
1439}
1440
1441static u32 clear_pmresrn_group(u32 val, int group)
1442{
1443	u32 mask;
1444	int group_shift;
1445
1446	group_shift = group * 8;
1447	mask = 0xff << group_shift;
1448	val &= ~mask;
1449
1450	/* Don't clear enable bit if entire region isn't disabled */
1451	if (val & ~PMRESRn_EN)
1452		return val |= PMRESRn_EN;
1453
1454	return 0;
1455}
1456
1457static void krait_clearpmu(u32 config_base)
1458{
1459	u32 val;
1460	u32 vval, fval;
1461	unsigned int region = EVENT_REGION(config_base);
1462	unsigned int group = EVENT_GROUP(config_base);
1463	bool venum_event = EVENT_VENUM(config_base);
1464
1465	if (venum_event) {
1466		venum_pre_pmresr(&vval, &fval);
1467		val = venum_read_pmresr();
1468		val = clear_pmresrn_group(val, group);
1469		venum_write_pmresr(val);
1470		venum_post_pmresr(vval, fval);
1471	} else {
1472		val = krait_read_pmresrn(region);
1473		val = clear_pmresrn_group(val, group);
1474		krait_write_pmresrn(region, val);
1475	}
1476}
1477
1478static void krait_pmu_disable_event(struct perf_event *event)
1479{
1480	unsigned long flags;
1481	struct hw_perf_event *hwc = &event->hw;
1482	int idx = hwc->idx;
1483	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1484	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1485
1486	/* Disable counter and interrupt */
1487	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1488
1489	/* Disable counter */
1490	armv7_pmnc_disable_counter(idx);
1491
1492	/*
1493	 * Clear pmresr code (if destined for PMNx counters)
1494	 */
1495	if (hwc->config_base & KRAIT_EVENT_MASK)
1496		krait_clearpmu(hwc->config_base);
1497
1498	/* Disable interrupt for this counter */
1499	armv7_pmnc_disable_intens(idx);
1500
1501	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1502}
1503
1504static void krait_pmu_enable_event(struct perf_event *event)
1505{
1506	unsigned long flags;
1507	struct hw_perf_event *hwc = &event->hw;
1508	int idx = hwc->idx;
1509	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1510	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1511
1512	/*
1513	 * Enable counter and interrupt, and set the counter to count
1514	 * the event that we're interested in.
1515	 */
1516	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1517
1518	/* Disable counter */
1519	armv7_pmnc_disable_counter(idx);
1520
1521	/*
1522	 * Set event (if destined for PMNx counters)
1523	 * We set the event for the cycle counter because we
1524	 * have the ability to perform event filtering.
1525	 */
1526	if (hwc->config_base & KRAIT_EVENT_MASK)
1527		krait_evt_setup(idx, hwc->config_base);
1528	else
1529		armv7_pmnc_write_evtsel(idx, hwc->config_base);
1530
1531	/* Enable interrupt for this counter */
1532	armv7_pmnc_enable_intens(idx);
1533
1534	/* Enable counter */
1535	armv7_pmnc_enable_counter(idx);
1536
1537	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1538}
1539
1540static void krait_pmu_reset(void *info)
1541{
1542	u32 vval, fval;
1543	struct arm_pmu *cpu_pmu = info;
1544	u32 idx, nb_cnt = cpu_pmu->num_events;
1545
1546	armv7pmu_reset(info);
1547
1548	/* Clear all pmresrs */
1549	krait_write_pmresrn(0, 0);
1550	krait_write_pmresrn(1, 0);
1551	krait_write_pmresrn(2, 0);
1552
1553	venum_pre_pmresr(&vval, &fval);
1554	venum_write_pmresr(0);
1555	venum_post_pmresr(vval, fval);
1556
1557	/* Reset PMxEVNCTCR to sane default */
1558	for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1559		armv7_pmnc_select_counter(idx);
1560		asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1561	}
1562
1563}
1564
1565static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1566			      unsigned int group)
1567{
1568	int bit;
1569	struct hw_perf_event *hwc = &event->hw;
1570	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1571
1572	if (hwc->config_base & VENUM_EVENT)
1573		bit = KRAIT_VPMRESR0_GROUP0;
1574	else
1575		bit = krait_get_pmresrn_event(region);
1576	bit -= krait_get_pmresrn_event(0);
1577	bit += group;
1578	/*
1579	 * Lower bits are reserved for use by the counters (see
1580	 * armv7pmu_get_event_idx() for more info)
1581	 */
1582	bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1583
1584	return bit;
1585}
1586
1587/*
1588 * We check for column exclusion constraints here.
1589 * Two events cant use the same group within a pmresr register.
1590 */
1591static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1592				   struct perf_event *event)
1593{
1594	int idx;
1595	int bit = -1;
1596	struct hw_perf_event *hwc = &event->hw;
1597	unsigned int region = EVENT_REGION(hwc->config_base);
1598	unsigned int code = EVENT_CODE(hwc->config_base);
1599	unsigned int group = EVENT_GROUP(hwc->config_base);
1600	bool venum_event = EVENT_VENUM(hwc->config_base);
1601	bool krait_event = EVENT_CPU(hwc->config_base);
1602
1603	if (venum_event || krait_event) {
1604		/* Ignore invalid events */
1605		if (group > 3 || region > 2)
1606			return -EINVAL;
1607		if (venum_event && (code & 0xe0))
1608			return -EINVAL;
1609
1610		bit = krait_event_to_bit(event, region, group);
1611		if (test_and_set_bit(bit, cpuc->used_mask))
1612			return -EAGAIN;
1613	}
1614
1615	idx = armv7pmu_get_event_idx(cpuc, event);
1616	if (idx < 0 && bit >= 0)
1617		clear_bit(bit, cpuc->used_mask);
1618
1619	return idx;
1620}
1621
1622static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1623				      struct perf_event *event)
1624{
1625	int bit;
1626	struct hw_perf_event *hwc = &event->hw;
1627	unsigned int region = EVENT_REGION(hwc->config_base);
1628	unsigned int group = EVENT_GROUP(hwc->config_base);
1629	bool venum_event = EVENT_VENUM(hwc->config_base);
1630	bool krait_event = EVENT_CPU(hwc->config_base);
1631
1632	if (venum_event || krait_event) {
1633		bit = krait_event_to_bit(event, region, group);
1634		clear_bit(bit, cpuc->used_mask);
1635	}
1636}
1637
1638static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1639{
1640	armv7pmu_init(cpu_pmu);
1641	cpu_pmu->name		= "armv7_krait";
1642	/* Some early versions of Krait don't support PC write events */
1643	if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1644				  "qcom,no-pc-write"))
1645		cpu_pmu->map_event = krait_map_event_no_branch;
1646	else
1647		cpu_pmu->map_event = krait_map_event;
1648	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1649	cpu_pmu->reset		= krait_pmu_reset;
1650	cpu_pmu->enable		= krait_pmu_enable_event;
1651	cpu_pmu->disable	= krait_pmu_disable_event;
1652	cpu_pmu->get_event_idx	= krait_pmu_get_event_idx;
1653	cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1654	return armv7_probe_num_events(cpu_pmu);
1655}
1656
1657/*
1658 * Scorpion Local Performance Monitor Register (LPMn)
1659 *
1660 *            31   30     24     16     8      0
1661 *            +--------------------------------+
1662 *  LPM0      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1663 *            +--------------------------------+
1664 *  LPM1      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1665 *            +--------------------------------+
1666 *  LPM2      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1667 *            +--------------------------------+
1668 *  L2LPM     | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 3
1669 *            +--------------------------------+
1670 *  VLPM      | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1671 *            +--------------------------------+
1672 *              EN | G=3  | G=2  | G=1  | G=0
1673 *
1674 *
1675 *  Event Encoding:
1676 *
1677 *      hwc->config_base = 0xNRCCG
1678 *
1679 *      N  = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1680 *      R  = region register
1681 *      CC = class of events the group G is choosing from
1682 *      G  = group or particular event
1683 *
1684 *  Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1685 *
1686 *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1687 *  unit, etc.) while the event code (CC) corresponds to a particular class of
1688 *  events (interrupts for example). An event code is broken down into
1689 *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1690 *  example).
1691 */
1692
1693static u32 scorpion_read_pmresrn(int n)
1694{
1695	u32 val;
1696
1697	switch (n) {
1698	case 0:
1699		asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1700		break;
1701	case 1:
1702		asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1703		break;
1704	case 2:
1705		asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1706		break;
1707	case 3:
1708		asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1709		break;
1710	default:
1711		BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1712	}
1713
1714	return val;
1715}
1716
1717static void scorpion_write_pmresrn(int n, u32 val)
1718{
1719	switch (n) {
1720	case 0:
1721		asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1722		break;
1723	case 1:
1724		asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1725		break;
1726	case 2:
1727		asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1728		break;
1729	case 3:
1730		asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1731		break;
1732	default:
1733		BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1734	}
1735}
1736
1737static u32 scorpion_get_pmresrn_event(unsigned int region)
1738{
1739	static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1740					     SCORPION_LPM1_GROUP0,
1741					     SCORPION_LPM2_GROUP0,
1742					     SCORPION_L2LPM_GROUP0 };
1743	return pmresrn_table[region];
1744}
1745
1746static void scorpion_evt_setup(int idx, u32 config_base)
1747{
1748	u32 val;
1749	u32 mask;
1750	u32 vval, fval;
1751	unsigned int region = EVENT_REGION(config_base);
1752	unsigned int group = EVENT_GROUP(config_base);
1753	unsigned int code = EVENT_CODE(config_base);
1754	unsigned int group_shift;
1755	bool venum_event = EVENT_VENUM(config_base);
1756
1757	group_shift = group * 8;
1758	mask = 0xff << group_shift;
1759
1760	/* Configure evtsel for the region and group */
1761	if (venum_event)
1762		val = SCORPION_VLPM_GROUP0;
1763	else
1764		val = scorpion_get_pmresrn_event(region);
1765	val += group;
1766	/* Mix in mode-exclusion bits */
1767	val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1768	armv7_pmnc_write_evtsel(idx, val);
1769
1770	asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1771
1772	if (venum_event) {
1773		venum_pre_pmresr(&vval, &fval);
1774		val = venum_read_pmresr();
1775		val &= ~mask;
1776		val |= code << group_shift;
1777		val |= PMRESRn_EN;
1778		venum_write_pmresr(val);
1779		venum_post_pmresr(vval, fval);
1780	} else {
1781		val = scorpion_read_pmresrn(region);
1782		val &= ~mask;
1783		val |= code << group_shift;
1784		val |= PMRESRn_EN;
1785		scorpion_write_pmresrn(region, val);
1786	}
1787}
1788
1789static void scorpion_clearpmu(u32 config_base)
1790{
1791	u32 val;
1792	u32 vval, fval;
1793	unsigned int region = EVENT_REGION(config_base);
1794	unsigned int group = EVENT_GROUP(config_base);
1795	bool venum_event = EVENT_VENUM(config_base);
1796
1797	if (venum_event) {
1798		venum_pre_pmresr(&vval, &fval);
1799		val = venum_read_pmresr();
1800		val = clear_pmresrn_group(val, group);
1801		venum_write_pmresr(val);
1802		venum_post_pmresr(vval, fval);
1803	} else {
1804		val = scorpion_read_pmresrn(region);
1805		val = clear_pmresrn_group(val, group);
1806		scorpion_write_pmresrn(region, val);
1807	}
1808}
1809
1810static void scorpion_pmu_disable_event(struct perf_event *event)
1811{
1812	unsigned long flags;
1813	struct hw_perf_event *hwc = &event->hw;
1814	int idx = hwc->idx;
1815	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1816	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1817
1818	/* Disable counter and interrupt */
1819	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1820
1821	/* Disable counter */
1822	armv7_pmnc_disable_counter(idx);
1823
1824	/*
1825	 * Clear pmresr code (if destined for PMNx counters)
1826	 */
1827	if (hwc->config_base & KRAIT_EVENT_MASK)
1828		scorpion_clearpmu(hwc->config_base);
1829
1830	/* Disable interrupt for this counter */
1831	armv7_pmnc_disable_intens(idx);
1832
1833	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1834}
1835
1836static void scorpion_pmu_enable_event(struct perf_event *event)
1837{
1838	unsigned long flags;
1839	struct hw_perf_event *hwc = &event->hw;
1840	int idx = hwc->idx;
1841	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1842	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1843
1844	/*
1845	 * Enable counter and interrupt, and set the counter to count
1846	 * the event that we're interested in.
1847	 */
1848	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1849
1850	/* Disable counter */
1851	armv7_pmnc_disable_counter(idx);
1852
1853	/*
1854	 * Set event (if destined for PMNx counters)
1855	 * We don't set the event for the cycle counter because we
1856	 * don't have the ability to perform event filtering.
1857	 */
1858	if (hwc->config_base & KRAIT_EVENT_MASK)
1859		scorpion_evt_setup(idx, hwc->config_base);
1860	else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1861		armv7_pmnc_write_evtsel(idx, hwc->config_base);
1862
1863	/* Enable interrupt for this counter */
1864	armv7_pmnc_enable_intens(idx);
1865
1866	/* Enable counter */
1867	armv7_pmnc_enable_counter(idx);
1868
1869	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1870}
1871
1872static void scorpion_pmu_reset(void *info)
1873{
1874	u32 vval, fval;
1875	struct arm_pmu *cpu_pmu = info;
1876	u32 idx, nb_cnt = cpu_pmu->num_events;
1877
1878	armv7pmu_reset(info);
1879
1880	/* Clear all pmresrs */
1881	scorpion_write_pmresrn(0, 0);
1882	scorpion_write_pmresrn(1, 0);
1883	scorpion_write_pmresrn(2, 0);
1884	scorpion_write_pmresrn(3, 0);
1885
1886	venum_pre_pmresr(&vval, &fval);
1887	venum_write_pmresr(0);
1888	venum_post_pmresr(vval, fval);
1889
1890	/* Reset PMxEVNCTCR to sane default */
1891	for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1892		armv7_pmnc_select_counter(idx);
1893		asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1894	}
1895}
1896
1897static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1898			      unsigned int group)
1899{
1900	int bit;
1901	struct hw_perf_event *hwc = &event->hw;
1902	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1903
1904	if (hwc->config_base & VENUM_EVENT)
1905		bit = SCORPION_VLPM_GROUP0;
1906	else
1907		bit = scorpion_get_pmresrn_event(region);
1908	bit -= scorpion_get_pmresrn_event(0);
1909	bit += group;
1910	/*
1911	 * Lower bits are reserved for use by the counters (see
1912	 * armv7pmu_get_event_idx() for more info)
1913	 */
1914	bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1915
1916	return bit;
1917}
1918
1919/*
1920 * We check for column exclusion constraints here.
1921 * Two events cant use the same group within a pmresr register.
1922 */
1923static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1924				   struct perf_event *event)
1925{
1926	int idx;
1927	int bit = -1;
1928	struct hw_perf_event *hwc = &event->hw;
1929	unsigned int region = EVENT_REGION(hwc->config_base);
1930	unsigned int group = EVENT_GROUP(hwc->config_base);
1931	bool venum_event = EVENT_VENUM(hwc->config_base);
1932	bool scorpion_event = EVENT_CPU(hwc->config_base);
1933
1934	if (venum_event || scorpion_event) {
1935		/* Ignore invalid events */
1936		if (group > 3 || region > 3)
1937			return -EINVAL;
1938
1939		bit = scorpion_event_to_bit(event, region, group);
1940		if (test_and_set_bit(bit, cpuc->used_mask))
1941			return -EAGAIN;
1942	}
1943
1944	idx = armv7pmu_get_event_idx(cpuc, event);
1945	if (idx < 0 && bit >= 0)
1946		clear_bit(bit, cpuc->used_mask);
1947
1948	return idx;
1949}
1950
1951static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1952				      struct perf_event *event)
1953{
1954	int bit;
1955	struct hw_perf_event *hwc = &event->hw;
1956	unsigned int region = EVENT_REGION(hwc->config_base);
1957	unsigned int group = EVENT_GROUP(hwc->config_base);
1958	bool venum_event = EVENT_VENUM(hwc->config_base);
1959	bool scorpion_event = EVENT_CPU(hwc->config_base);
1960
1961	if (venum_event || scorpion_event) {
1962		bit = scorpion_event_to_bit(event, region, group);
1963		clear_bit(bit, cpuc->used_mask);
1964	}
1965}
1966
1967static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1968{
1969	armv7pmu_init(cpu_pmu);
1970	cpu_pmu->name		= "armv7_scorpion";
1971	cpu_pmu->map_event	= scorpion_map_event;
1972	cpu_pmu->reset		= scorpion_pmu_reset;
1973	cpu_pmu->enable		= scorpion_pmu_enable_event;
1974	cpu_pmu->disable	= scorpion_pmu_disable_event;
1975	cpu_pmu->get_event_idx	= scorpion_pmu_get_event_idx;
1976	cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1977	return armv7_probe_num_events(cpu_pmu);
1978}
1979
1980static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1981{
1982	armv7pmu_init(cpu_pmu);
1983	cpu_pmu->name		= "armv7_scorpion_mp";
1984	cpu_pmu->map_event	= scorpion_map_event;
1985	cpu_pmu->reset		= scorpion_pmu_reset;
1986	cpu_pmu->enable		= scorpion_pmu_enable_event;
1987	cpu_pmu->disable	= scorpion_pmu_disable_event;
1988	cpu_pmu->get_event_idx	= scorpion_pmu_get_event_idx;
1989	cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1990	return armv7_probe_num_events(cpu_pmu);
1991}
1992
1993static const struct of_device_id armv7_pmu_of_device_ids[] = {
1994	{.compatible = "arm,cortex-a17-pmu",	.data = armv7_a17_pmu_init},
1995	{.compatible = "arm,cortex-a15-pmu",	.data = armv7_a15_pmu_init},
1996	{.compatible = "arm,cortex-a12-pmu",	.data = armv7_a12_pmu_init},
1997	{.compatible = "arm,cortex-a9-pmu",	.data = armv7_a9_pmu_init},
1998	{.compatible = "arm,cortex-a8-pmu",	.data = armv7_a8_pmu_init},
1999	{.compatible = "arm,cortex-a7-pmu",	.data = armv7_a7_pmu_init},
2000	{.compatible = "arm,cortex-a5-pmu",	.data = armv7_a5_pmu_init},
2001	{.compatible = "qcom,krait-pmu",	.data = krait_pmu_init},
2002	{.compatible = "qcom,scorpion-pmu",	.data = scorpion_pmu_init},
2003	{.compatible = "qcom,scorpion-mp-pmu",	.data = scorpion_mp_pmu_init},
2004	{},
2005};
2006
2007static const struct pmu_probe_info armv7_pmu_probe_table[] = {
2008	ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
2009	ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
2010	{ /* sentinel value */ }
2011};
2012
2013
2014static int armv7_pmu_device_probe(struct platform_device *pdev)
2015{
2016	return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
2017				    armv7_pmu_probe_table);
2018}
2019
2020static struct platform_driver armv7_pmu_driver = {
2021	.driver		= {
2022		.name	= "armv7-pmu",
2023		.of_match_table = armv7_pmu_of_device_ids,
2024	},
2025	.probe		= armv7_pmu_device_probe,
2026};
2027
2028static int __init register_armv7_pmu_driver(void)
2029{
2030	return platform_driver_register(&armv7_pmu_driver);
2031}
2032device_initcall(register_armv7_pmu_driver);
2033#endif	/* CONFIG_CPU_V7 */