1/*
   2 * Linux performance counter support for MIPS.
   3 *
   4 * Copyright (C) 2010 MIPS Technologies, Inc.
   5 * Copyright (C) 2011 Cavium Networks, Inc.
   6 * Author: Deng-Cheng Zhu
   7 *
   8 * This code is based on the implementation for ARM, which is in turn
   9 * based on the sparc64 perf event code and the x86 code. Performance
  10 * counter access is based on the MIPS Oprofile code. And the callchain
  11 * support references the code of MIPS stacktrace.c.
  12 *
  13 * This program is free software; you can redistribute it and/or modify
  14 * it under the terms of the GNU General Public License version 2 as
  15 * published by the Free Software Foundation.
  16 */
  17
  18#include <linux/cpumask.h>
  19#include <linux/interrupt.h>
  20#include <linux/smp.h>
  21#include <linux/kernel.h>
  22#include <linux/perf_event.h>
  23#include <linux/uaccess.h>
  24
  25#include <asm/irq.h>
  26#include <asm/irq_regs.h>
  27#include <asm/stacktrace.h>
  28#include <asm/time.h> /* For perf_irq */
  29
  30#define MIPS_MAX_HWEVENTS 4
  31#define MIPS_TCS_PER_COUNTER 2
  32#define MIPS_CPUID_TO_COUNTER_MASK (MIPS_TCS_PER_COUNTER - 1)
  33
  34struct cpu_hw_events {
  35	/* Array of events on this cpu. */
  36	struct perf_event	*events[MIPS_MAX_HWEVENTS];
  37
  38	/*
  39	 * Set the bit (indexed by the counter number) when the counter
  40	 * is used for an event.
  41	 */
  42	unsigned long		used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
  43
  44	/*
  45	 * Software copy of the control register for each performance counter.
  46	 * MIPS CPUs vary in performance counters. They use this differently,
  47	 * and even may not use it.
  48	 */
  49	unsigned int		saved_ctrl[MIPS_MAX_HWEVENTS];
  50};
  51DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  52	.saved_ctrl = {0},
  53};
  54
  55/* The description of MIPS performance events. */
  56struct mips_perf_event {
  57	unsigned int event_id;
  58	/*
  59	 * MIPS performance counters are indexed starting from 0.
  60	 * CNTR_EVEN indicates the indexes of the counters to be used are
  61	 * even numbers.
  62	 */
  63	unsigned int cntr_mask;
  64	#define CNTR_EVEN	0x55555555
  65	#define CNTR_ODD	0xaaaaaaaa
  66	#define CNTR_ALL	0xffffffff
  67#ifdef CONFIG_MIPS_MT_SMP
  68	enum {
  69		T  = 0,
  70		V  = 1,
  71		P  = 2,
  72	} range;
  73#else
  74	#define T
  75	#define V
  76	#define P
  77#endif
  78};
  79
  80static struct mips_perf_event raw_event;
  81static DEFINE_MUTEX(raw_event_mutex);
  82
  83#define C(x) PERF_COUNT_HW_CACHE_##x
  84
  85struct mips_pmu {
  86	u64		max_period;
  87	u64		valid_count;
  88	u64		overflow;
  89	const char	*name;
  90	int		irq;
  91	u64		(*read_counter)(unsigned int idx);
  92	void		(*write_counter)(unsigned int idx, u64 val);
  93	const struct mips_perf_event *(*map_raw_event)(u64 config);
  94	const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
  95	const struct mips_perf_event (*cache_event_map)
  96				[PERF_COUNT_HW_CACHE_MAX]
  97				[PERF_COUNT_HW_CACHE_OP_MAX]
  98				[PERF_COUNT_HW_CACHE_RESULT_MAX];
  99	unsigned int	num_counters;
 100};
 101
 102static struct mips_pmu mipspmu;
 103
 104#define M_CONFIG1_PC	(1 << 4)
 105
 106#define M_PERFCTL_EXL			(1	<<  0)
 107#define M_PERFCTL_KERNEL		(1	<<  1)
 108#define M_PERFCTL_SUPERVISOR		(1	<<  2)
 109#define M_PERFCTL_USER			(1	<<  3)
 110#define M_PERFCTL_INTERRUPT_ENABLE	(1	<<  4)
 111#define M_PERFCTL_EVENT(event)		(((event) & 0x3ff)  << 5)
 112#define M_PERFCTL_VPEID(vpe)		((vpe)	  << 16)
 113
 114#ifdef CONFIG_CPU_BMIPS5000
 115#define M_PERFCTL_MT_EN(filter)		0
 116#else /* !CONFIG_CPU_BMIPS5000 */
 117#define M_PERFCTL_MT_EN(filter)		((filter) << 20)
 118#endif /* CONFIG_CPU_BMIPS5000 */
 119
 120#define	   M_TC_EN_ALL			M_PERFCTL_MT_EN(0)
 121#define	   M_TC_EN_VPE			M_PERFCTL_MT_EN(1)
 122#define	   M_TC_EN_TC			M_PERFCTL_MT_EN(2)
 123#define M_PERFCTL_TCID(tcid)		((tcid)	  << 22)
 124#define M_PERFCTL_WIDE			(1	<< 30)
 125#define M_PERFCTL_MORE			(1	<< 31)
 126#define M_PERFCTL_TC			(1	<< 30)
 127
 128#define M_PERFCTL_COUNT_EVENT_WHENEVER	(M_PERFCTL_EXL |		\
 129					M_PERFCTL_KERNEL |		\
 130					M_PERFCTL_USER |		\
 131					M_PERFCTL_SUPERVISOR |		\
 132					M_PERFCTL_INTERRUPT_ENABLE)
 133
 134#ifdef CONFIG_MIPS_MT_SMP
 135#define M_PERFCTL_CONFIG_MASK		0x3fff801f
 136#else
 137#define M_PERFCTL_CONFIG_MASK		0x1f
 138#endif
 139#define M_PERFCTL_EVENT_MASK		0xfe0
 140
 141
 142#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
 143static int cpu_has_mipsmt_pertccounters;
 144
 145static DEFINE_RWLOCK(pmuint_rwlock);
 146
 147#if defined(CONFIG_CPU_BMIPS5000)
 148#define vpe_id()	(cpu_has_mipsmt_pertccounters ? \
 149			 0 : (smp_processor_id() & MIPS_CPUID_TO_COUNTER_MASK))
 150#else
 151/*
 152 * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because
 153 * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs.
 154 */
 155#define vpe_id()	(cpu_has_mipsmt_pertccounters ? \
 156			 0 : smp_processor_id())
 157#endif
 158
 159/* Copied from op_model_mipsxx.c */
 160static unsigned int vpe_shift(void)
 161{
 162	if (num_possible_cpus() > 1)
 163		return 1;
 164
 165	return 0;
 166}
 167
 168static unsigned int counters_total_to_per_cpu(unsigned int counters)
 169{
 170	return counters >> vpe_shift();
 171}
 172
 173#else /* !CONFIG_MIPS_PERF_SHARED_TC_COUNTERS */
 174#define vpe_id()	0
 175
 176#endif /* CONFIG_MIPS_PERF_SHARED_TC_COUNTERS */
 177
 178static void resume_local_counters(void);
 179static void pause_local_counters(void);
 180static irqreturn_t mipsxx_pmu_handle_irq(int, void *);
 181static int mipsxx_pmu_handle_shared_irq(void);
 182
 183static unsigned int mipsxx_pmu_swizzle_perf_idx(unsigned int idx)
 184{
 185	if (vpe_id() == 1)
 186		idx = (idx + 2) & 3;
 187	return idx;
 188}
 189
 190static u64 mipsxx_pmu_read_counter(unsigned int idx)
 191{
 192	idx = mipsxx_pmu_swizzle_perf_idx(idx);
 193
 194	switch (idx) {
 195	case 0:
 196		/*
 197		 * The counters are unsigned, we must cast to truncate
 198		 * off the high bits.
 199		 */
 200		return (u32)read_c0_perfcntr0();
 201	case 1:
 202		return (u32)read_c0_perfcntr1();
 203	case 2:
 204		return (u32)read_c0_perfcntr2();
 205	case 3:
 206		return (u32)read_c0_perfcntr3();
 207	default:
 208		WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
 209		return 0;
 210	}
 211}
 212
 213static u64 mipsxx_pmu_read_counter_64(unsigned int idx)
 214{
 215	idx = mipsxx_pmu_swizzle_perf_idx(idx);
 216
 217	switch (idx) {
 218	case 0:
 219		return read_c0_perfcntr0_64();
 220	case 1:
 221		return read_c0_perfcntr1_64();
 222	case 2:
 223		return read_c0_perfcntr2_64();
 224	case 3:
 225		return read_c0_perfcntr3_64();
 226	default:
 227		WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
 228		return 0;
 229	}
 230}
 231
 232static void mipsxx_pmu_write_counter(unsigned int idx, u64 val)
 233{
 234	idx = mipsxx_pmu_swizzle_perf_idx(idx);
 235
 236	switch (idx) {
 237	case 0:
 238		write_c0_perfcntr0(val);
 239		return;
 240	case 1:
 241		write_c0_perfcntr1(val);
 242		return;
 243	case 2:
 244		write_c0_perfcntr2(val);
 245		return;
 246	case 3:
 247		write_c0_perfcntr3(val);
 248		return;
 249	}
 250}
 251
 252static void mipsxx_pmu_write_counter_64(unsigned int idx, u64 val)
 253{
 254	idx = mipsxx_pmu_swizzle_perf_idx(idx);
 255
 256	switch (idx) {
 257	case 0:
 258		write_c0_perfcntr0_64(val);
 259		return;
 260	case 1:
 261		write_c0_perfcntr1_64(val);
 262		return;
 263	case 2:
 264		write_c0_perfcntr2_64(val);
 265		return;
 266	case 3:
 267		write_c0_perfcntr3_64(val);
 268		return;
 269	}
 270}
 271
 272static unsigned int mipsxx_pmu_read_control(unsigned int idx)
 273{
 274	idx = mipsxx_pmu_swizzle_perf_idx(idx);
 275
 276	switch (idx) {
 277	case 0:
 278		return read_c0_perfctrl0();
 279	case 1:
 280		return read_c0_perfctrl1();
 281	case 2:
 282		return read_c0_perfctrl2();
 283	case 3:
 284		return read_c0_perfctrl3();
 285	default:
 286		WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
 287		return 0;
 288	}
 289}
 290
 291static void mipsxx_pmu_write_control(unsigned int idx, unsigned int val)
 292{
 293	idx = mipsxx_pmu_swizzle_perf_idx(idx);
 294
 295	switch (idx) {
 296	case 0:
 297		write_c0_perfctrl0(val);
 298		return;
 299	case 1:
 300		write_c0_perfctrl1(val);
 301		return;
 302	case 2:
 303		write_c0_perfctrl2(val);
 304		return;
 305	case 3:
 306		write_c0_perfctrl3(val);
 307		return;
 308	}
 309}
 310
 311static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc,
 312				    struct hw_perf_event *hwc)
 313{
 314	int i;
 315
 316	/*
 317	 * We only need to care the counter mask. The range has been
 318	 * checked definitely.
 319	 */
 320	unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff;
 321
 322	for (i = mipspmu.num_counters - 1; i >= 0; i--) {
 323		/*
 324		 * Note that some MIPS perf events can be counted by both
 325		 * even and odd counters, wheresas many other are only by
 326		 * even _or_ odd counters. This introduces an issue that
 327		 * when the former kind of event takes the counter the
 328		 * latter kind of event wants to use, then the "counter
 329		 * allocation" for the latter event will fail. In fact if
 330		 * they can be dynamically swapped, they both feel happy.
 331		 * But here we leave this issue alone for now.
 332		 */
 333		if (test_bit(i, &cntr_mask) &&
 334			!test_and_set_bit(i, cpuc->used_mask))
 335			return i;
 336	}
 337
 338	return -EAGAIN;
 339}
 340
 341static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx)
 342{
 343	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 344
 345	WARN_ON(idx < 0 || idx >= mipspmu.num_counters);
 346
 347	cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) |
 348		(evt->config_base & M_PERFCTL_CONFIG_MASK) |
 349		/* Make sure interrupt enabled. */
 350		M_PERFCTL_INTERRUPT_ENABLE;
 351	if (IS_ENABLED(CONFIG_CPU_BMIPS5000))
 352		/* enable the counter for the calling thread */
 353		cpuc->saved_ctrl[idx] |=
 354			(1 << (12 + vpe_id())) | M_PERFCTL_TC;
 355
 356	/*
 357	 * We do not actually let the counter run. Leave it until start().
 358	 */
 359}
 360
 361static void mipsxx_pmu_disable_event(int idx)
 362{
 363	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 364	unsigned long flags;
 365
 366	WARN_ON(idx < 0 || idx >= mipspmu.num_counters);
 367
 368	local_irq_save(flags);
 369	cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) &
 370		~M_PERFCTL_COUNT_EVENT_WHENEVER;
 371	mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]);
 372	local_irq_restore(flags);
 373}
 374
 375static int mipspmu_event_set_period(struct perf_event *event,
 376				    struct hw_perf_event *hwc,
 377				    int idx)
 378{
 379	u64 left = local64_read(&hwc->period_left);
 380	u64 period = hwc->sample_period;
 381	int ret = 0;
 382
 383	if (unlikely((left + period) & (1ULL << 63))) {
 384		/* left underflowed by more than period. */
 385		left = period;
 386		local64_set(&hwc->period_left, left);
 387		hwc->last_period = period;
 388		ret = 1;
 389	} else	if (unlikely((left + period) <= period)) {
 390		/* left underflowed by less than period. */
 391		left += period;
 392		local64_set(&hwc->period_left, left);
 393		hwc->last_period = period;
 394		ret = 1;
 395	}
 396
 397	if (left > mipspmu.max_period) {
 398		left = mipspmu.max_period;
 399		local64_set(&hwc->period_left, left);
 400	}
 401
 402	local64_set(&hwc->prev_count, mipspmu.overflow - left);
 403
 404	mipspmu.write_counter(idx, mipspmu.overflow - left);
 405
 406	perf_event_update_userpage(event);
 407
 408	return ret;
 409}
 410
 411static void mipspmu_event_update(struct perf_event *event,
 412				 struct hw_perf_event *hwc,
 413				 int idx)
 414{
 415	u64 prev_raw_count, new_raw_count;
 416	u64 delta;
 417
 418again:
 419	prev_raw_count = local64_read(&hwc->prev_count);
 420	new_raw_count = mipspmu.read_counter(idx);
 421
 422	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 423				new_raw_count) != prev_raw_count)
 424		goto again;
 425
 426	delta = new_raw_count - prev_raw_count;
 427
 428	local64_add(delta, &event->count);
 429	local64_sub(delta, &hwc->period_left);
 430}
 431
 432static void mipspmu_start(struct perf_event *event, int flags)
 433{
 434	struct hw_perf_event *hwc = &event->hw;
 435
 436	if (flags & PERF_EF_RELOAD)
 437		WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 438
 439	hwc->state = 0;
 440
 441	/* Set the period for the event. */
 442	mipspmu_event_set_period(event, hwc, hwc->idx);
 443
 444	/* Enable the event. */
 445	mipsxx_pmu_enable_event(hwc, hwc->idx);
 446}
 447
 448static void mipspmu_stop(struct perf_event *event, int flags)
 449{
 450	struct hw_perf_event *hwc = &event->hw;
 451
 452	if (!(hwc->state & PERF_HES_STOPPED)) {
 453		/* We are working on a local event. */
 454		mipsxx_pmu_disable_event(hwc->idx);
 455		barrier();
 456		mipspmu_event_update(event, hwc, hwc->idx);
 457		hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 458	}
 459}
 460
 461static int mipspmu_add(struct perf_event *event, int flags)
 462{
 463	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 464	struct hw_perf_event *hwc = &event->hw;
 465	int idx;
 466	int err = 0;
 467
 468	perf_pmu_disable(event->pmu);
 469
 470	/* To look for a free counter for this event. */
 471	idx = mipsxx_pmu_alloc_counter(cpuc, hwc);
 472	if (idx < 0) {
 473		err = idx;
 474		goto out;
 475	}
 476
 477	/*
 478	 * If there is an event in the counter we are going to use then
 479	 * make sure it is disabled.
 480	 */
 481	event->hw.idx = idx;
 482	mipsxx_pmu_disable_event(idx);
 483	cpuc->events[idx] = event;
 484
 485	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 486	if (flags & PERF_EF_START)
 487		mipspmu_start(event, PERF_EF_RELOAD);
 488
 489	/* Propagate our changes to the userspace mapping. */
 490	perf_event_update_userpage(event);
 491
 492out:
 493	perf_pmu_enable(event->pmu);
 494	return err;
 495}
 496
 497static void mipspmu_del(struct perf_event *event, int flags)
 498{
 499	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 500	struct hw_perf_event *hwc = &event->hw;
 501	int idx = hwc->idx;
 502
 503	WARN_ON(idx < 0 || idx >= mipspmu.num_counters);
 504
 505	mipspmu_stop(event, PERF_EF_UPDATE);
 506	cpuc->events[idx] = NULL;
 507	clear_bit(idx, cpuc->used_mask);
 508
 509	perf_event_update_userpage(event);
 510}
 511
 512static void mipspmu_read(struct perf_event *event)
 513{
 514	struct hw_perf_event *hwc = &event->hw;
 515
 516	/* Don't read disabled counters! */
 517	if (hwc->idx < 0)
 518		return;
 519
 520	mipspmu_event_update(event, hwc, hwc->idx);
 521}
 522
 523static void mipspmu_enable(struct pmu *pmu)
 524{
 525#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
 526	write_unlock(&pmuint_rwlock);
 527#endif
 528	resume_local_counters();
 529}
 530
 531/*
 532 * MIPS performance counters can be per-TC. The control registers can
 533 * not be directly accessed across CPUs. Hence if we want to do global
 534 * control, we need cross CPU calls. on_each_cpu() can help us, but we
 535 * can not make sure this function is called with interrupts enabled. So
 536 * here we pause local counters and then grab a rwlock and leave the
 537 * counters on other CPUs alone. If any counter interrupt raises while
 538 * we own the write lock, simply pause local counters on that CPU and
 539 * spin in the handler. Also we know we won't be switched to another
 540 * CPU after pausing local counters and before grabbing the lock.
 541 */
 542static void mipspmu_disable(struct pmu *pmu)
 543{
 544	pause_local_counters();
 545#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
 546	write_lock(&pmuint_rwlock);
 547#endif
 548}
 549
 550static atomic_t active_events = ATOMIC_INIT(0);
 551static DEFINE_MUTEX(pmu_reserve_mutex);
 552static int (*save_perf_irq)(void);
 553
 554static int mipspmu_get_irq(void)
 555{
 556	int err;
 557
 558	if (mipspmu.irq >= 0) {
 559		/* Request my own irq handler. */
 560		err = request_irq(mipspmu.irq, mipsxx_pmu_handle_irq,
 561				  IRQF_PERCPU | IRQF_NOBALANCING |
 562				  IRQF_NO_THREAD | IRQF_NO_SUSPEND |
 563				  IRQF_SHARED,
 564				  "mips_perf_pmu", &mipspmu);
 565		if (err) {
 566			pr_warn("Unable to request IRQ%d for MIPS performance counters!\n",
 567				mipspmu.irq);
 568		}
 569	} else if (cp0_perfcount_irq < 0) {
 570		/*
 571		 * We are sharing the irq number with the timer interrupt.
 572		 */
 573		save_perf_irq = perf_irq;
 574		perf_irq = mipsxx_pmu_handle_shared_irq;
 575		err = 0;
 576	} else {
 577		pr_warn("The platform hasn't properly defined its interrupt controller\n");
 
 578		err = -ENOENT;
 579	}
 580
 581	return err;
 582}
 583
 584static void mipspmu_free_irq(void)
 585{
 586	if (mipspmu.irq >= 0)
 587		free_irq(mipspmu.irq, &mipspmu);
 588	else if (cp0_perfcount_irq < 0)
 589		perf_irq = save_perf_irq;
 590}
 591
 592/*
 593 * mipsxx/rm9000/loongson2 have different performance counters, they have
 594 * specific low-level init routines.
 595 */
 596static void reset_counters(void *arg);
 597static int __hw_perf_event_init(struct perf_event *event);
 598
 599static void hw_perf_event_destroy(struct perf_event *event)
 600{
 601	if (atomic_dec_and_mutex_lock(&active_events,
 602				&pmu_reserve_mutex)) {
 603		/*
 604		 * We must not call the destroy function with interrupts
 605		 * disabled.
 606		 */
 607		on_each_cpu(reset_counters,
 608			(void *)(long)mipspmu.num_counters, 1);
 609		mipspmu_free_irq();
 610		mutex_unlock(&pmu_reserve_mutex);
 611	}
 612}
 613
 614static int mipspmu_event_init(struct perf_event *event)
 615{
 616	int err = 0;
 617
 618	/* does not support taken branch sampling */
 619	if (has_branch_stack(event))
 620		return -EOPNOTSUPP;
 621
 622	switch (event->attr.type) {
 623	case PERF_TYPE_RAW:
 624	case PERF_TYPE_HARDWARE:
 625	case PERF_TYPE_HW_CACHE:
 626		break;
 627
 628	default:
 629		return -ENOENT;
 630	}
 631
 632	if (event->cpu >= nr_cpumask_bits ||
 633	    (event->cpu >= 0 && !cpu_online(event->cpu)))
 634		return -ENODEV;
 635
 636	if (!atomic_inc_not_zero(&active_events)) {
 637		mutex_lock(&pmu_reserve_mutex);
 638		if (atomic_read(&active_events) == 0)
 639			err = mipspmu_get_irq();
 640
 641		if (!err)
 642			atomic_inc(&active_events);
 643		mutex_unlock(&pmu_reserve_mutex);
 644	}
 645
 646	if (err)
 647		return err;
 648
 649	return __hw_perf_event_init(event);
 650}
 651
 652static struct pmu pmu = {
 653	.pmu_enable	= mipspmu_enable,
 654	.pmu_disable	= mipspmu_disable,
 655	.event_init	= mipspmu_event_init,
 656	.add		= mipspmu_add,
 657	.del		= mipspmu_del,
 658	.start		= mipspmu_start,
 659	.stop		= mipspmu_stop,
 660	.read		= mipspmu_read,
 661};
 662
 663static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev)
 664{
 665/*
 666 * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for
 667 * event_id.
 668 */
 669#ifdef CONFIG_MIPS_MT_SMP
 670	return ((unsigned int)pev->range << 24) |
 671		(pev->cntr_mask & 0xffff00) |
 672		(pev->event_id & 0xff);
 673#else
 674	return (pev->cntr_mask & 0xffff00) |
 675		(pev->event_id & 0xff);
 676#endif
 677}
 678
 679static const struct mips_perf_event *mipspmu_map_general_event(int idx)
 680{
 681
 682	if ((*mipspmu.general_event_map)[idx].cntr_mask == 0)
 683		return ERR_PTR(-EOPNOTSUPP);
 684	return &(*mipspmu.general_event_map)[idx];
 685}
 686
 687static const struct mips_perf_event *mipspmu_map_cache_event(u64 config)
 688{
 689	unsigned int cache_type, cache_op, cache_result;
 690	const struct mips_perf_event *pev;
 691
 692	cache_type = (config >> 0) & 0xff;
 693	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 694		return ERR_PTR(-EINVAL);
 695
 696	cache_op = (config >> 8) & 0xff;
 697	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 698		return ERR_PTR(-EINVAL);
 699
 700	cache_result = (config >> 16) & 0xff;
 701	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 702		return ERR_PTR(-EINVAL);
 703
 704	pev = &((*mipspmu.cache_event_map)
 705					[cache_type]
 706					[cache_op]
 707					[cache_result]);
 708
 709	if (pev->cntr_mask == 0)
 710		return ERR_PTR(-EOPNOTSUPP);
 711
 712	return pev;
 713
 714}
 715
 716static int validate_group(struct perf_event *event)
 717{
 718	struct perf_event *sibling, *leader = event->group_leader;
 719	struct cpu_hw_events fake_cpuc;
 720
 721	memset(&fake_cpuc, 0, sizeof(fake_cpuc));
 722
 723	if (mipsxx_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0)
 724		return -EINVAL;
 725
 726	list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
 727		if (mipsxx_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0)
 728			return -EINVAL;
 729	}
 730
 731	if (mipsxx_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0)
 732		return -EINVAL;
 733
 734	return 0;
 735}
 736
 737/* This is needed by specific irq handlers in perf_event_*.c */
 738static void handle_associated_event(struct cpu_hw_events *cpuc,
 739				    int idx, struct perf_sample_data *data,
 740				    struct pt_regs *regs)
 741{
 742	struct perf_event *event = cpuc->events[idx];
 743	struct hw_perf_event *hwc = &event->hw;
 744
 745	mipspmu_event_update(event, hwc, idx);
 746	data->period = event->hw.last_period;
 747	if (!mipspmu_event_set_period(event, hwc, idx))
 748		return;
 749
 750	if (perf_event_overflow(event, data, regs))
 751		mipsxx_pmu_disable_event(idx);
 752}
 753
 754
 755static int __n_counters(void)
 756{
 757	if (!(read_c0_config1() & M_CONFIG1_PC))
 758		return 0;
 759	if (!(read_c0_perfctrl0() & M_PERFCTL_MORE))
 760		return 1;
 761	if (!(read_c0_perfctrl1() & M_PERFCTL_MORE))
 762		return 2;
 763	if (!(read_c0_perfctrl2() & M_PERFCTL_MORE))
 764		return 3;
 765
 766	return 4;
 767}
 768
 769static int n_counters(void)
 770{
 771	int counters;
 772
 773	switch (current_cpu_type()) {
 774	case CPU_R10000:
 775		counters = 2;
 776		break;
 777
 778	case CPU_R12000:
 779	case CPU_R14000:
 780	case CPU_R16000:
 781		counters = 4;
 782		break;
 783
 784	default:
 785		counters = __n_counters();
 786	}
 787
 788	return counters;
 789}
 790
 791static void reset_counters(void *arg)
 792{
 793	int counters = (int)(long)arg;
 794	switch (counters) {
 795	case 4:
 796		mipsxx_pmu_write_control(3, 0);
 797		mipspmu.write_counter(3, 0);
 798	case 3:
 799		mipsxx_pmu_write_control(2, 0);
 800		mipspmu.write_counter(2, 0);
 801	case 2:
 802		mipsxx_pmu_write_control(1, 0);
 803		mipspmu.write_counter(1, 0);
 804	case 1:
 805		mipsxx_pmu_write_control(0, 0);
 806		mipspmu.write_counter(0, 0);
 807	}
 808}
 809
 810/* 24K/34K/1004K/interAptiv/loongson1 cores share the same event map. */
 811static const struct mips_perf_event mipsxxcore_event_map
 812				[PERF_COUNT_HW_MAX] = {
 813	[PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P },
 814	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
 815	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_EVEN, T },
 816	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T },
 817};
 818
 819/* 74K/proAptiv core has different branch event code. */
 820static const struct mips_perf_event mipsxxcore_event_map2
 821				[PERF_COUNT_HW_MAX] = {
 822	[PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P },
 823	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
 824	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x27, CNTR_EVEN, T },
 825	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x27, CNTR_ODD, T },
 826};
 827
 828static const struct mips_perf_event loongson3_event_map[PERF_COUNT_HW_MAX] = {
 829	[PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN },
 830	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, CNTR_ODD },
 831	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x01, CNTR_EVEN },
 832	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x01, CNTR_ODD },
 833};
 834
 835static const struct mips_perf_event octeon_event_map[PERF_COUNT_HW_MAX] = {
 836	[PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL },
 837	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x03, CNTR_ALL },
 838	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x2b, CNTR_ALL },
 839	[PERF_COUNT_HW_CACHE_MISSES] = { 0x2e, CNTR_ALL	 },
 840	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x08, CNTR_ALL },
 841	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x09, CNTR_ALL },
 842	[PERF_COUNT_HW_BUS_CYCLES] = { 0x25, CNTR_ALL },
 843};
 844
 845static const struct mips_perf_event bmips5000_event_map
 846				[PERF_COUNT_HW_MAX] = {
 847	[PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, T },
 848	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
 849	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T },
 850};
 851
 852static const struct mips_perf_event xlp_event_map[PERF_COUNT_HW_MAX] = {
 853	[PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL },
 854	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x18, CNTR_ALL }, /* PAPI_TOT_INS */
 855	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */
 856	[PERF_COUNT_HW_CACHE_MISSES] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */
 857	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x1b, CNTR_ALL }, /* PAPI_BR_CN */
 858	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x1c, CNTR_ALL }, /* PAPI_BR_MSP */
 859};
 860
 861/* 24K/34K/1004K/interAptiv/loongson1 cores share the same cache event map. */
 862static const struct mips_perf_event mipsxxcore_cache_map
 863				[PERF_COUNT_HW_CACHE_MAX]
 864				[PERF_COUNT_HW_CACHE_OP_MAX]
 865				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 866[C(L1D)] = {
 867	/*
 868	 * Like some other architectures (e.g. ARM), the performance
 869	 * counters don't differentiate between read and write
 870	 * accesses/misses, so this isn't strictly correct, but it's the
 871	 * best we can do. Writes and reads get combined.
 872	 */
 873	[C(OP_READ)] = {
 874		[C(RESULT_ACCESS)]	= { 0x0a, CNTR_EVEN, T },
 875		[C(RESULT_MISS)]	= { 0x0b, CNTR_EVEN | CNTR_ODD, T },
 876	},
 877	[C(OP_WRITE)] = {
 878		[C(RESULT_ACCESS)]	= { 0x0a, CNTR_EVEN, T },
 879		[C(RESULT_MISS)]	= { 0x0b, CNTR_EVEN | CNTR_ODD, T },
 880	},
 881},
 882[C(L1I)] = {
 883	[C(OP_READ)] = {
 884		[C(RESULT_ACCESS)]	= { 0x09, CNTR_EVEN, T },
 885		[C(RESULT_MISS)]	= { 0x09, CNTR_ODD, T },
 886	},
 887	[C(OP_WRITE)] = {
 888		[C(RESULT_ACCESS)]	= { 0x09, CNTR_EVEN, T },
 889		[C(RESULT_MISS)]	= { 0x09, CNTR_ODD, T },
 890	},
 891	[C(OP_PREFETCH)] = {
 892		[C(RESULT_ACCESS)]	= { 0x14, CNTR_EVEN, T },
 893		/*
 894		 * Note that MIPS has only "hit" events countable for
 895		 * the prefetch operation.
 896		 */
 897	},
 898},
 899[C(LL)] = {
 900	[C(OP_READ)] = {
 901		[C(RESULT_ACCESS)]	= { 0x15, CNTR_ODD, P },
 902		[C(RESULT_MISS)]	= { 0x16, CNTR_EVEN, P },
 903	},
 904	[C(OP_WRITE)] = {
 905		[C(RESULT_ACCESS)]	= { 0x15, CNTR_ODD, P },
 906		[C(RESULT_MISS)]	= { 0x16, CNTR_EVEN, P },
 907	},
 908},
 909[C(DTLB)] = {
 910	[C(OP_READ)] = {
 911		[C(RESULT_ACCESS)]	= { 0x06, CNTR_EVEN, T },
 912		[C(RESULT_MISS)]	= { 0x06, CNTR_ODD, T },
 913	},
 914	[C(OP_WRITE)] = {
 915		[C(RESULT_ACCESS)]	= { 0x06, CNTR_EVEN, T },
 916		[C(RESULT_MISS)]	= { 0x06, CNTR_ODD, T },
 917	},
 918},
 919[C(ITLB)] = {
 920	[C(OP_READ)] = {
 921		[C(RESULT_ACCESS)]	= { 0x05, CNTR_EVEN, T },
 922		[C(RESULT_MISS)]	= { 0x05, CNTR_ODD, T },
 923	},
 924	[C(OP_WRITE)] = {
 925		[C(RESULT_ACCESS)]	= { 0x05, CNTR_EVEN, T },
 926		[C(RESULT_MISS)]	= { 0x05, CNTR_ODD, T },
 927	},
 928},
 929[C(BPU)] = {
 930	/* Using the same code for *HW_BRANCH* */
 931	[C(OP_READ)] = {
 932		[C(RESULT_ACCESS)]	= { 0x02, CNTR_EVEN, T },
 933		[C(RESULT_MISS)]	= { 0x02, CNTR_ODD, T },
 934	},
 935	[C(OP_WRITE)] = {
 936		[C(RESULT_ACCESS)]	= { 0x02, CNTR_EVEN, T },
 937		[C(RESULT_MISS)]	= { 0x02, CNTR_ODD, T },
 938	},
 939},
 940};
 941
 942/* 74K/proAptiv core has completely different cache event map. */
 943static const struct mips_perf_event mipsxxcore_cache_map2
 944				[PERF_COUNT_HW_CACHE_MAX]
 945				[PERF_COUNT_HW_CACHE_OP_MAX]
 946				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 947[C(L1D)] = {
 948	/*
 949	 * Like some other architectures (e.g. ARM), the performance
 950	 * counters don't differentiate between read and write
 951	 * accesses/misses, so this isn't strictly correct, but it's the
 952	 * best we can do. Writes and reads get combined.
 953	 */
 954	[C(OP_READ)] = {
 955		[C(RESULT_ACCESS)]	= { 0x17, CNTR_ODD, T },
 956		[C(RESULT_MISS)]	= { 0x18, CNTR_ODD, T },
 957	},
 958	[C(OP_WRITE)] = {
 959		[C(RESULT_ACCESS)]	= { 0x17, CNTR_ODD, T },
 960		[C(RESULT_MISS)]	= { 0x18, CNTR_ODD, T },
 961	},
 962},
 963[C(L1I)] = {
 964	[C(OP_READ)] = {
 965		[C(RESULT_ACCESS)]	= { 0x06, CNTR_EVEN, T },
 966		[C(RESULT_MISS)]	= { 0x06, CNTR_ODD, T },
 967	},
 968	[C(OP_WRITE)] = {
 969		[C(RESULT_ACCESS)]	= { 0x06, CNTR_EVEN, T },
 970		[C(RESULT_MISS)]	= { 0x06, CNTR_ODD, T },
 971	},
 972	[C(OP_PREFETCH)] = {
 973		[C(RESULT_ACCESS)]	= { 0x34, CNTR_EVEN, T },
 974		/*
 975		 * Note that MIPS has only "hit" events countable for
 976		 * the prefetch operation.
 977		 */
 978	},
 979},
 980[C(LL)] = {
 981	[C(OP_READ)] = {
 982		[C(RESULT_ACCESS)]	= { 0x1c, CNTR_ODD, P },
 983		[C(RESULT_MISS)]	= { 0x1d, CNTR_EVEN, P },
 984	},
 985	[C(OP_WRITE)] = {
 986		[C(RESULT_ACCESS)]	= { 0x1c, CNTR_ODD, P },
 987		[C(RESULT_MISS)]	= { 0x1d, CNTR_EVEN, P },
 988	},
 989},
 990/*
 991 * 74K core does not have specific DTLB events. proAptiv core has
 992 * "speculative" DTLB events which are numbered 0x63 (even/odd) and
 993 * not included here. One can use raw events if really needed.
 994 */
 995[C(ITLB)] = {
 996	[C(OP_READ)] = {
 997		[C(RESULT_ACCESS)]	= { 0x04, CNTR_EVEN, T },
 998		[C(RESULT_MISS)]	= { 0x04, CNTR_ODD, T },
 999	},
1000	[C(OP_WRITE)] = {
1001		[C(RESULT_ACCESS)]	= { 0x04, CNTR_EVEN, T },
1002		[C(RESULT_MISS)]	= { 0x04, CNTR_ODD, T },
1003	},
1004},
1005[C(BPU)] = {
1006	/* Using the same code for *HW_BRANCH* */
1007	[C(OP_READ)] = {
1008		[C(RESULT_ACCESS)]	= { 0x27, CNTR_EVEN, T },
1009		[C(RESULT_MISS)]	= { 0x27, CNTR_ODD, T },
1010	},
1011	[C(OP_WRITE)] = {
1012		[C(RESULT_ACCESS)]	= { 0x27, CNTR_EVEN, T },
1013		[C(RESULT_MISS)]	= { 0x27, CNTR_ODD, T },
1014	},
1015},
1016};
1017
1018static const struct mips_perf_event loongson3_cache_map
1019				[PERF_COUNT_HW_CACHE_MAX]
1020				[PERF_COUNT_HW_CACHE_OP_MAX]
1021				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1022[C(L1D)] = {
1023	/*
1024	 * Like some other architectures (e.g. ARM), the performance
1025	 * counters don't differentiate between read and write
1026	 * accesses/misses, so this isn't strictly correct, but it's the
1027	 * best we can do. Writes and reads get combined.
1028	 */
1029	[C(OP_READ)] = {
1030		[C(RESULT_MISS)]        = { 0x04, CNTR_ODD },
1031	},
1032	[C(OP_WRITE)] = {
1033		[C(RESULT_MISS)]        = { 0x04, CNTR_ODD },
1034	},
1035},
1036[C(L1I)] = {
1037	[C(OP_READ)] = {
1038		[C(RESULT_MISS)]        = { 0x04, CNTR_EVEN },
1039	},
1040	[C(OP_WRITE)] = {
1041		[C(RESULT_MISS)]        = { 0x04, CNTR_EVEN },
1042	},
1043},
1044[C(DTLB)] = {
1045	[C(OP_READ)] = {
1046		[C(RESULT_MISS)]        = { 0x09, CNTR_ODD },
1047	},
1048	[C(OP_WRITE)] = {
1049		[C(RESULT_MISS)]        = { 0x09, CNTR_ODD },
1050	},
1051},
1052[C(ITLB)] = {
1053	[C(OP_READ)] = {
1054		[C(RESULT_MISS)]        = { 0x0c, CNTR_ODD },
1055	},
1056	[C(OP_WRITE)] = {
1057		[C(RESULT_MISS)]        = { 0x0c, CNTR_ODD },
1058	},
1059},
1060[C(BPU)] = {
1061	/* Using the same code for *HW_BRANCH* */
1062	[C(OP_READ)] = {
1063		[C(RESULT_ACCESS)]      = { 0x02, CNTR_EVEN },
1064		[C(RESULT_MISS)]        = { 0x02, CNTR_ODD },
1065	},
1066	[C(OP_WRITE)] = {
1067		[C(RESULT_ACCESS)]      = { 0x02, CNTR_EVEN },
1068		[C(RESULT_MISS)]        = { 0x02, CNTR_ODD },
1069	},
1070},
1071};
1072
1073/* BMIPS5000 */
1074static const struct mips_perf_event bmips5000_cache_map
1075				[PERF_COUNT_HW_CACHE_MAX]
1076				[PERF_COUNT_HW_CACHE_OP_MAX]
1077				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1078[C(L1D)] = {
1079	/*
1080	 * Like some other architectures (e.g. ARM), the performance
1081	 * counters don't differentiate between read and write
1082	 * accesses/misses, so this isn't strictly correct, but it's the
1083	 * best we can do. Writes and reads get combined.
1084	 */
1085	[C(OP_READ)] = {
1086		[C(RESULT_ACCESS)]	= { 12, CNTR_EVEN, T },
1087		[C(RESULT_MISS)]	= { 12, CNTR_ODD, T },
1088	},
1089	[C(OP_WRITE)] = {
1090		[C(RESULT_ACCESS)]	= { 12, CNTR_EVEN, T },
1091		[C(RESULT_MISS)]	= { 12, CNTR_ODD, T },
1092	},
1093},
1094[C(L1I)] = {
1095	[C(OP_READ)] = {
1096		[C(RESULT_ACCESS)]	= { 10, CNTR_EVEN, T },
1097		[C(RESULT_MISS)]	= { 10, CNTR_ODD, T },
1098	},
1099	[C(OP_WRITE)] = {
1100		[C(RESULT_ACCESS)]	= { 10, CNTR_EVEN, T },
1101		[C(RESULT_MISS)]	= { 10, CNTR_ODD, T },
1102	},
1103	[C(OP_PREFETCH)] = {
1104		[C(RESULT_ACCESS)]	= { 23, CNTR_EVEN, T },
1105		/*
1106		 * Note that MIPS has only "hit" events countable for
1107		 * the prefetch operation.
1108		 */
1109	},
1110},
1111[C(LL)] = {
1112	[C(OP_READ)] = {
1113		[C(RESULT_ACCESS)]	= { 28, CNTR_EVEN, P },
1114		[C(RESULT_MISS)]	= { 28, CNTR_ODD, P },
1115	},
1116	[C(OP_WRITE)] = {
1117		[C(RESULT_ACCESS)]	= { 28, CNTR_EVEN, P },
1118		[C(RESULT_MISS)]	= { 28, CNTR_ODD, P },
1119	},
1120},
1121[C(BPU)] = {
1122	/* Using the same code for *HW_BRANCH* */
1123	[C(OP_READ)] = {
1124		[C(RESULT_MISS)]	= { 0x02, CNTR_ODD, T },
1125	},
1126	[C(OP_WRITE)] = {
1127		[C(RESULT_MISS)]	= { 0x02, CNTR_ODD, T },
1128	},
1129},
1130};
1131
1132
1133static const struct mips_perf_event octeon_cache_map
1134				[PERF_COUNT_HW_CACHE_MAX]
1135				[PERF_COUNT_HW_CACHE_OP_MAX]
1136				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1137[C(L1D)] = {
1138	[C(OP_READ)] = {
1139		[C(RESULT_ACCESS)]	= { 0x2b, CNTR_ALL },
1140		[C(RESULT_MISS)]	= { 0x2e, CNTR_ALL },
1141	},
1142	[C(OP_WRITE)] = {
1143		[C(RESULT_ACCESS)]	= { 0x30, CNTR_ALL },
1144	},
1145},
1146[C(L1I)] = {
1147	[C(OP_READ)] = {
1148		[C(RESULT_ACCESS)]	= { 0x18, CNTR_ALL },
1149	},
1150	[C(OP_PREFETCH)] = {
1151		[C(RESULT_ACCESS)]	= { 0x19, CNTR_ALL },
1152	},
1153},
1154[C(DTLB)] = {
1155	/*
1156	 * Only general DTLB misses are counted use the same event for
1157	 * read and write.
1158	 */
1159	[C(OP_READ)] = {
1160		[C(RESULT_MISS)]	= { 0x35, CNTR_ALL },
1161	},
1162	[C(OP_WRITE)] = {
1163		[C(RESULT_MISS)]	= { 0x35, CNTR_ALL },
1164	},
1165},
1166[C(ITLB)] = {
1167	[C(OP_READ)] = {
1168		[C(RESULT_MISS)]	= { 0x37, CNTR_ALL },
1169	},
1170},
1171};
1172
1173static const struct mips_perf_event xlp_cache_map
1174				[PERF_COUNT_HW_CACHE_MAX]
1175				[PERF_COUNT_HW_CACHE_OP_MAX]
1176				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1177[C(L1D)] = {
1178	[C(OP_READ)] = {
1179		[C(RESULT_ACCESS)]	= { 0x31, CNTR_ALL }, /* PAPI_L1_DCR */
1180		[C(RESULT_MISS)]	= { 0x30, CNTR_ALL }, /* PAPI_L1_LDM */
1181	},
1182	[C(OP_WRITE)] = {
1183		[C(RESULT_ACCESS)]	= { 0x2f, CNTR_ALL }, /* PAPI_L1_DCW */
1184		[C(RESULT_MISS)]	= { 0x2e, CNTR_ALL }, /* PAPI_L1_STM */
1185	},
1186},
1187[C(L1I)] = {
1188	[C(OP_READ)] = {
1189		[C(RESULT_ACCESS)]	= { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */
1190		[C(RESULT_MISS)]	= { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */
1191	},
1192},
1193[C(LL)] = {
1194	[C(OP_READ)] = {
1195		[C(RESULT_ACCESS)]	= { 0x35, CNTR_ALL }, /* PAPI_L2_DCR */
1196		[C(RESULT_MISS)]	= { 0x37, CNTR_ALL }, /* PAPI_L2_LDM */
1197	},
1198	[C(OP_WRITE)] = {
1199		[C(RESULT_ACCESS)]	= { 0x34, CNTR_ALL }, /* PAPI_L2_DCA */
1200		[C(RESULT_MISS)]	= { 0x36, CNTR_ALL }, /* PAPI_L2_DCM */
1201	},
1202},
1203[C(DTLB)] = {
1204	/*
1205	 * Only general DTLB misses are counted use the same event for
1206	 * read and write.
1207	 */
1208	[C(OP_READ)] = {
1209		[C(RESULT_MISS)]	= { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */
1210	},
1211	[C(OP_WRITE)] = {
1212		[C(RESULT_MISS)]	= { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */
1213	},
1214},
1215[C(ITLB)] = {
1216	[C(OP_READ)] = {
1217		[C(RESULT_MISS)]	= { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */
1218	},
1219	[C(OP_WRITE)] = {
1220		[C(RESULT_MISS)]	= { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */
1221	},
1222},
1223[C(BPU)] = {
1224	[C(OP_READ)] = {
1225		[C(RESULT_MISS)]	= { 0x25, CNTR_ALL },
1226	},
1227},
1228};
1229
1230#ifdef CONFIG_MIPS_MT_SMP
1231static void check_and_calc_range(struct perf_event *event,
1232				 const struct mips_perf_event *pev)
1233{
1234	struct hw_perf_event *hwc = &event->hw;
1235
1236	if (event->cpu >= 0) {
1237		if (pev->range > V) {
1238			/*
1239			 * The user selected an event that is processor
1240			 * wide, while expecting it to be VPE wide.
1241			 */
1242			hwc->config_base |= M_TC_EN_ALL;
1243		} else {
1244			/*
1245			 * FIXME: cpu_data[event->cpu].vpe_id reports 0
1246			 * for both CPUs.
1247			 */
1248			hwc->config_base |= M_PERFCTL_VPEID(event->cpu);
1249			hwc->config_base |= M_TC_EN_VPE;
1250		}
1251	} else
1252		hwc->config_base |= M_TC_EN_ALL;
1253}
1254#else
1255static void check_and_calc_range(struct perf_event *event,
1256				 const struct mips_perf_event *pev)
1257{
1258}
1259#endif
1260
1261static int __hw_perf_event_init(struct perf_event *event)
1262{
1263	struct perf_event_attr *attr = &event->attr;
1264	struct hw_perf_event *hwc = &event->hw;
1265	const struct mips_perf_event *pev;
1266	int err;
1267
1268	/* Returning MIPS event descriptor for generic perf event. */
1269	if (PERF_TYPE_HARDWARE == event->attr.type) {
1270		if (event->attr.config >= PERF_COUNT_HW_MAX)
1271			return -EINVAL;
1272		pev = mipspmu_map_general_event(event->attr.config);
1273	} else if (PERF_TYPE_HW_CACHE == event->attr.type) {
1274		pev = mipspmu_map_cache_event(event->attr.config);
1275	} else if (PERF_TYPE_RAW == event->attr.type) {
1276		/* We are working on the global raw event. */
1277		mutex_lock(&raw_event_mutex);
1278		pev = mipspmu.map_raw_event(event->attr.config);
1279	} else {
1280		/* The event type is not (yet) supported. */
1281		return -EOPNOTSUPP;
1282	}
1283
1284	if (IS_ERR(pev)) {
1285		if (PERF_TYPE_RAW == event->attr.type)
1286			mutex_unlock(&raw_event_mutex);
1287		return PTR_ERR(pev);
1288	}
1289
1290	/*
1291	 * We allow max flexibility on how each individual counter shared
1292	 * by the single CPU operates (the mode exclusion and the range).
1293	 */
1294	hwc->config_base = M_PERFCTL_INTERRUPT_ENABLE;
1295
1296	/* Calculate range bits and validate it. */
1297	if (num_possible_cpus() > 1)
1298		check_and_calc_range(event, pev);
1299
1300	hwc->event_base = mipspmu_perf_event_encode(pev);
1301	if (PERF_TYPE_RAW == event->attr.type)
1302		mutex_unlock(&raw_event_mutex);
1303
1304	if (!attr->exclude_user)
1305		hwc->config_base |= M_PERFCTL_USER;
1306	if (!attr->exclude_kernel) {
1307		hwc->config_base |= M_PERFCTL_KERNEL;
1308		/* MIPS kernel mode: KSU == 00b || EXL == 1 || ERL == 1 */
1309		hwc->config_base |= M_PERFCTL_EXL;
1310	}
1311	if (!attr->exclude_hv)
1312		hwc->config_base |= M_PERFCTL_SUPERVISOR;
1313
1314	hwc->config_base &= M_PERFCTL_CONFIG_MASK;
1315	/*
1316	 * The event can belong to another cpu. We do not assign a local
1317	 * counter for it for now.
1318	 */
1319	hwc->idx = -1;
1320	hwc->config = 0;
1321
1322	if (!hwc->sample_period) {
1323		hwc->sample_period  = mipspmu.max_period;
1324		hwc->last_period    = hwc->sample_period;
1325		local64_set(&hwc->period_left, hwc->sample_period);
1326	}
1327
1328	err = 0;
1329	if (event->group_leader != event)
1330		err = validate_group(event);
1331
1332	event->destroy = hw_perf_event_destroy;
1333
1334	if (err)
1335		event->destroy(event);
1336
1337	return err;
1338}
1339
1340static void pause_local_counters(void)
1341{
1342	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1343	int ctr = mipspmu.num_counters;
1344	unsigned long flags;
1345
1346	local_irq_save(flags);
1347	do {
1348		ctr--;
1349		cpuc->saved_ctrl[ctr] = mipsxx_pmu_read_control(ctr);
1350		mipsxx_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] &
1351					 ~M_PERFCTL_COUNT_EVENT_WHENEVER);
1352	} while (ctr > 0);
1353	local_irq_restore(flags);
1354}
1355
1356static void resume_local_counters(void)
1357{
1358	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1359	int ctr = mipspmu.num_counters;
1360
1361	do {
1362		ctr--;
1363		mipsxx_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]);
1364	} while (ctr > 0);
1365}
1366
1367static int mipsxx_pmu_handle_shared_irq(void)
1368{
1369	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1370	struct perf_sample_data data;
1371	unsigned int counters = mipspmu.num_counters;
1372	u64 counter;
1373	int handled = IRQ_NONE;
1374	struct pt_regs *regs;
1375
1376	if (cpu_has_perf_cntr_intr_bit && !(read_c0_cause() & CAUSEF_PCI))
1377		return handled;
1378	/*
1379	 * First we pause the local counters, so that when we are locked
1380	 * here, the counters are all paused. When it gets locked due to
1381	 * perf_disable(), the timer interrupt handler will be delayed.
1382	 *
1383	 * See also mipsxx_pmu_start().
1384	 */
1385	pause_local_counters();
1386#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
1387	read_lock(&pmuint_rwlock);
1388#endif
1389
1390	regs = get_irq_regs();
1391
1392	perf_sample_data_init(&data, 0, 0);
1393
1394	switch (counters) {
1395#define HANDLE_COUNTER(n)						\
1396	case n + 1:							\
1397		if (test_bit(n, cpuc->used_mask)) {			\
1398			counter = mipspmu.read_counter(n);		\
1399			if (counter & mipspmu.overflow) {		\
1400				handle_associated_event(cpuc, n, &data, regs); \
1401				handled = IRQ_HANDLED;			\
1402			}						\
1403		}
1404	HANDLE_COUNTER(3)
1405	HANDLE_COUNTER(2)
1406	HANDLE_COUNTER(1)
1407	HANDLE_COUNTER(0)
1408	}
1409
1410	/*
1411	 * Do all the work for the pending perf events. We can do this
1412	 * in here because the performance counter interrupt is a regular
1413	 * interrupt, not NMI.
1414	 */
1415	if (handled == IRQ_HANDLED)
1416		irq_work_run();
1417
1418#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
1419	read_unlock(&pmuint_rwlock);
1420#endif
1421	resume_local_counters();
1422	return handled;
1423}
1424
1425static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev)
1426{
1427	return mipsxx_pmu_handle_shared_irq();
1428}
1429
1430/* 24K */
1431#define IS_BOTH_COUNTERS_24K_EVENT(b)					\
1432	((b) == 0 || (b) == 1 || (b) == 11)
1433
1434/* 34K */
1435#define IS_BOTH_COUNTERS_34K_EVENT(b)					\
1436	((b) == 0 || (b) == 1 || (b) == 11)
1437#ifdef CONFIG_MIPS_MT_SMP
1438#define IS_RANGE_P_34K_EVENT(r, b)					\
1439	((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 ||		\
1440	 (b) == 25 || (b) == 39 || (r) == 44 || (r) == 174 ||		\
1441	 (r) == 176 || ((b) >= 50 && (b) <= 55) ||			\
1442	 ((b) >= 64 && (b) <= 67))
1443#define IS_RANGE_V_34K_EVENT(r) ((r) == 47)
1444#endif
1445
1446/* 74K */
1447#define IS_BOTH_COUNTERS_74K_EVENT(b)					\
1448	((b) == 0 || (b) == 1)
1449
1450/* proAptiv */
1451#define IS_BOTH_COUNTERS_PROAPTIV_EVENT(b)				\
1452	((b) == 0 || (b) == 1)
1453/* P5600 */
1454#define IS_BOTH_COUNTERS_P5600_EVENT(b)					\
1455	((b) == 0 || (b) == 1)
1456
1457/* 1004K */
1458#define IS_BOTH_COUNTERS_1004K_EVENT(b)					\
1459	((b) == 0 || (b) == 1 || (b) == 11)
1460#ifdef CONFIG_MIPS_MT_SMP
1461#define IS_RANGE_P_1004K_EVENT(r, b)					\
1462	((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 ||		\
1463	 (b) == 25 || (b) == 36 || (b) == 39 || (r) == 44 ||		\
1464	 (r) == 174 || (r) == 176 || ((b) >= 50 && (b) <= 59) ||	\
1465	 (r) == 188 || (b) == 61 || (b) == 62 ||			\
1466	 ((b) >= 64 && (b) <= 67))
1467#define IS_RANGE_V_1004K_EVENT(r)	((r) == 47)
1468#endif
1469
1470/* interAptiv */
1471#define IS_BOTH_COUNTERS_INTERAPTIV_EVENT(b)				\
1472	((b) == 0 || (b) == 1 || (b) == 11)
1473#ifdef CONFIG_MIPS_MT_SMP
1474/* The P/V/T info is not provided for "(b) == 38" in SUM, assume P. */
1475#define IS_RANGE_P_INTERAPTIV_EVENT(r, b)				\
1476	((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 ||		\
1477	 (b) == 25 || (b) == 36 || (b) == 38 || (b) == 39 ||		\
1478	 (r) == 44 || (r) == 174 || (r) == 176 || ((b) >= 50 &&		\
1479	 (b) <= 59) || (r) == 188 || (b) == 61 || (b) == 62 ||		\
1480	 ((b) >= 64 && (b) <= 67))
1481#define IS_RANGE_V_INTERAPTIV_EVENT(r)	((r) == 47 || (r) == 175)
1482#endif
1483
1484/* BMIPS5000 */
1485#define IS_BOTH_COUNTERS_BMIPS5000_EVENT(b)				\
1486	((b) == 0 || (b) == 1)
1487
1488
1489/*
1490 * For most cores the user can use 0-255 raw events, where 0-127 for the events
1491 * of even counters, and 128-255 for odd counters. Note that bit 7 is used to
1492 * indicate the even/odd bank selector. So, for example, when user wants to take
1493 * the Event Num of 15 for odd counters (by referring to the user manual), then
1494 * 128 needs to be added to 15 as the input for the event config, i.e., 143 (0x8F)
1495 * to be used.
1496 *
1497 * Some newer cores have even more events, in which case the user can use raw
1498 * events 0-511, where 0-255 are for the events of even counters, and 256-511
1499 * are for odd counters, so bit 8 is used to indicate the even/odd bank selector.
1500 */
1501static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config)
1502{
1503	/* currently most cores have 7-bit event numbers */
1504	unsigned int raw_id = config & 0xff;
1505	unsigned int base_id = raw_id & 0x7f;
1506
 
 
1507	switch (current_cpu_type()) {
1508	case CPU_24K:
1509		if (IS_BOTH_COUNTERS_24K_EVENT(base_id))
1510			raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
1511		else
1512			raw_event.cntr_mask =
1513				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
1514#ifdef CONFIG_MIPS_MT_SMP
1515		/*
1516		 * This is actually doing nothing. Non-multithreading
1517		 * CPUs will not check and calculate the range.
1518		 */
1519		raw_event.range = P;
1520#endif
1521		break;
1522	case CPU_34K:
1523		if (IS_BOTH_COUNTERS_34K_EVENT(base_id))
1524			raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
1525		else
1526			raw_event.cntr_mask =
1527				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
1528#ifdef CONFIG_MIPS_MT_SMP
1529		if (IS_RANGE_P_34K_EVENT(raw_id, base_id))
1530			raw_event.range = P;
1531		else if (unlikely(IS_RANGE_V_34K_EVENT(raw_id)))
1532			raw_event.range = V;
1533		else
1534			raw_event.range = T;
1535#endif
1536		break;
1537	case CPU_74K:
1538	case CPU_1074K:
1539		if (IS_BOTH_COUNTERS_74K_EVENT(base_id))
1540			raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
1541		else
1542			raw_event.cntr_mask =
1543				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
1544#ifdef CONFIG_MIPS_MT_SMP
1545		raw_event.range = P;
1546#endif
1547		break;
1548	case CPU_PROAPTIV:
1549		if (IS_BOTH_COUNTERS_PROAPTIV_EVENT(base_id))
1550			raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
1551		else
1552			raw_event.cntr_mask =
1553				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
1554#ifdef CONFIG_MIPS_MT_SMP
1555		raw_event.range = P;
1556#endif
1557		break;
1558	case CPU_P5600:
1559	case CPU_I6400:
1560		/* 8-bit event numbers */
1561		raw_id = config & 0x1ff;
1562		base_id = raw_id & 0xff;
1563		if (IS_BOTH_COUNTERS_P5600_EVENT(base_id))
1564			raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
1565		else
1566			raw_event.cntr_mask =
1567				raw_id > 255 ? CNTR_ODD : CNTR_EVEN;
1568#ifdef CONFIG_MIPS_MT_SMP
1569		raw_event.range = P;
1570#endif
1571		break;
1572	case CPU_1004K:
1573		if (IS_BOTH_COUNTERS_1004K_EVENT(base_id))
1574			raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
1575		else
1576			raw_event.cntr_mask =
1577				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
1578#ifdef CONFIG_MIPS_MT_SMP
1579		if (IS_RANGE_P_1004K_EVENT(raw_id, base_id))
1580			raw_event.range = P;
1581		else if (unlikely(IS_RANGE_V_1004K_EVENT(raw_id)))
1582			raw_event.range = V;
1583		else
1584			raw_event.range = T;
1585#endif
1586		break;
1587	case CPU_INTERAPTIV:
1588		if (IS_BOTH_COUNTERS_INTERAPTIV_EVENT(base_id))
1589			raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
1590		else
1591			raw_event.cntr_mask =
1592				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
1593#ifdef CONFIG_MIPS_MT_SMP
1594		if (IS_RANGE_P_INTERAPTIV_EVENT(raw_id, base_id))
1595			raw_event.range = P;
1596		else if (unlikely(IS_RANGE_V_INTERAPTIV_EVENT(raw_id)))
1597			raw_event.range = V;
1598		else
1599			raw_event.range = T;
1600#endif
1601		break;
1602	case CPU_BMIPS5000:
1603		if (IS_BOTH_COUNTERS_BMIPS5000_EVENT(base_id))
1604			raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
1605		else
1606			raw_event.cntr_mask =
1607				raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
1608		break;
1609	case CPU_LOONGSON3:
1610		raw_event.cntr_mask = raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
1611	break;
1612	}
1613
1614	raw_event.event_id = base_id;
1615
1616	return &raw_event;
1617}
1618
1619static const struct mips_perf_event *octeon_pmu_map_raw_event(u64 config)
1620{
1621	unsigned int raw_id = config & 0xff;
1622	unsigned int base_id = raw_id & 0x7f;
1623
1624
1625	raw_event.cntr_mask = CNTR_ALL;
1626	raw_event.event_id = base_id;
1627
1628	if (current_cpu_type() == CPU_CAVIUM_OCTEON2) {
1629		if (base_id > 0x42)
1630			return ERR_PTR(-EOPNOTSUPP);
1631	} else {
1632		if (base_id > 0x3a)
1633			return ERR_PTR(-EOPNOTSUPP);
1634	}
1635
1636	switch (base_id) {
1637	case 0x00:
1638	case 0x0f:
1639	case 0x1e:
1640	case 0x1f:
1641	case 0x2f:
1642	case 0x34:
1643	case 0x3b ... 0x3f:
1644		return ERR_PTR(-EOPNOTSUPP);
1645	default:
1646		break;
1647	}
1648
1649	return &raw_event;
1650}
1651
1652static const struct mips_perf_event *xlp_pmu_map_raw_event(u64 config)
1653{
1654	unsigned int raw_id = config & 0xff;
1655
1656	/* Only 1-63 are defined */
1657	if ((raw_id < 0x01) || (raw_id > 0x3f))
1658		return ERR_PTR(-EOPNOTSUPP);
1659
1660	raw_event.cntr_mask = CNTR_ALL;
1661	raw_event.event_id = raw_id;
1662
1663	return &raw_event;
1664}
1665
1666static int __init
1667init_hw_perf_events(void)
1668{
1669	int counters, irq;
1670	int counter_bits;
1671
1672	pr_info("Performance counters: ");
1673
1674	counters = n_counters();
1675	if (counters == 0) {
1676		pr_cont("No available PMU.\n");
1677		return -ENODEV;
1678	}
1679
1680#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
1681	cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19);
1682	if (!cpu_has_mipsmt_pertccounters)
1683		counters = counters_total_to_per_cpu(counters);
1684#endif
1685
1686	if (get_c0_perfcount_int)
1687		irq = get_c0_perfcount_int();
1688	else if (cp0_perfcount_irq >= 0)
1689		irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
1690	else
1691		irq = -1;
 
 
 
 
 
 
 
 
 
 
1692
1693	mipspmu.map_raw_event = mipsxx_pmu_map_raw_event;
1694
1695	switch (current_cpu_type()) {
1696	case CPU_24K:
1697		mipspmu.name = "mips/24K";
1698		mipspmu.general_event_map = &mipsxxcore_event_map;
1699		mipspmu.cache_event_map = &mipsxxcore_cache_map;
1700		break;
1701	case CPU_34K:
1702		mipspmu.name = "mips/34K";
1703		mipspmu.general_event_map = &mipsxxcore_event_map;
1704		mipspmu.cache_event_map = &mipsxxcore_cache_map;
1705		break;
1706	case CPU_74K:
1707		mipspmu.name = "mips/74K";
1708		mipspmu.general_event_map = &mipsxxcore_event_map2;
1709		mipspmu.cache_event_map = &mipsxxcore_cache_map2;
1710		break;
1711	case CPU_PROAPTIV:
1712		mipspmu.name = "mips/proAptiv";
1713		mipspmu.general_event_map = &mipsxxcore_event_map2;
1714		mipspmu.cache_event_map = &mipsxxcore_cache_map2;
1715		break;
1716	case CPU_P5600:
1717		mipspmu.name = "mips/P5600";
1718		mipspmu.general_event_map = &mipsxxcore_event_map2;
1719		mipspmu.cache_event_map = &mipsxxcore_cache_map2;
1720		break;
1721	case CPU_I6400:
1722		mipspmu.name = "mips/I6400";
1723		mipspmu.general_event_map = &mipsxxcore_event_map2;
1724		mipspmu.cache_event_map = &mipsxxcore_cache_map2;
1725		break;
1726	case CPU_1004K:
1727		mipspmu.name = "mips/1004K";
1728		mipspmu.general_event_map = &mipsxxcore_event_map;
1729		mipspmu.cache_event_map = &mipsxxcore_cache_map;
1730		break;
1731	case CPU_1074K:
1732		mipspmu.name = "mips/1074K";
1733		mipspmu.general_event_map = &mipsxxcore_event_map;
1734		mipspmu.cache_event_map = &mipsxxcore_cache_map;
1735		break;
1736	case CPU_INTERAPTIV:
1737		mipspmu.name = "mips/interAptiv";
1738		mipspmu.general_event_map = &mipsxxcore_event_map;
1739		mipspmu.cache_event_map = &mipsxxcore_cache_map;
1740		break;
1741	case CPU_LOONGSON1:
1742		mipspmu.name = "mips/loongson1";
1743		mipspmu.general_event_map = &mipsxxcore_event_map;
1744		mipspmu.cache_event_map = &mipsxxcore_cache_map;
1745		break;
1746	case CPU_LOONGSON3:
1747		mipspmu.name = "mips/loongson3";
1748		mipspmu.general_event_map = &loongson3_event_map;
1749		mipspmu.cache_event_map = &loongson3_cache_map;
1750		break;
1751	case CPU_CAVIUM_OCTEON:
1752	case CPU_CAVIUM_OCTEON_PLUS:
1753	case CPU_CAVIUM_OCTEON2:
1754		mipspmu.name = "octeon";
1755		mipspmu.general_event_map = &octeon_event_map;
1756		mipspmu.cache_event_map = &octeon_cache_map;
1757		mipspmu.map_raw_event = octeon_pmu_map_raw_event;
1758		break;
1759	case CPU_BMIPS5000:
1760		mipspmu.name = "BMIPS5000";
1761		mipspmu.general_event_map = &bmips5000_event_map;
1762		mipspmu.cache_event_map = &bmips5000_cache_map;
1763		break;
1764	case CPU_XLP:
1765		mipspmu.name = "xlp";
1766		mipspmu.general_event_map = &xlp_event_map;
1767		mipspmu.cache_event_map = &xlp_cache_map;
1768		mipspmu.map_raw_event = xlp_pmu_map_raw_event;
1769		break;
1770	default:
1771		pr_cont("Either hardware does not support performance "
1772			"counters, or not yet implemented.\n");
1773		return -ENODEV;
1774	}
1775
1776	mipspmu.num_counters = counters;
1777	mipspmu.irq = irq;
1778
1779	if (read_c0_perfctrl0() & M_PERFCTL_WIDE) {
1780		mipspmu.max_period = (1ULL << 63) - 1;
1781		mipspmu.valid_count = (1ULL << 63) - 1;
1782		mipspmu.overflow = 1ULL << 63;
1783		mipspmu.read_counter = mipsxx_pmu_read_counter_64;
1784		mipspmu.write_counter = mipsxx_pmu_write_counter_64;
1785		counter_bits = 64;
1786	} else {
1787		mipspmu.max_period = (1ULL << 31) - 1;
1788		mipspmu.valid_count = (1ULL << 31) - 1;
1789		mipspmu.overflow = 1ULL << 31;
1790		mipspmu.read_counter = mipsxx_pmu_read_counter;
1791		mipspmu.write_counter = mipsxx_pmu_write_counter;
1792		counter_bits = 32;
1793	}
1794
1795	on_each_cpu(reset_counters, (void *)(long)counters, 1);
1796
1797	pr_cont("%s PMU enabled, %d %d-bit counters available to each "
1798		"CPU, irq %d%s\n", mipspmu.name, counters, counter_bits, irq,
1799		irq < 0 ? " (share with timer interrupt)" : "");
1800
1801	perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1802
1803	return 0;
1804}
1805early_initcall(init_hw_perf_events);