1/*
   2 * Performance events x86 architecture code
   3 *
   4 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6 *  Copyright (C) 2009 Jaswinder Singh Rajput
   7 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9 *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10 *  Copyright (C) 2009 Google, Inc., Stephane Eranian
  11 *
  12 *  For licencing details see kernel-base/COPYING
  13 */
  14
  15#include <linux/perf_event.h>
  16#include <linux/capability.h>
  17#include <linux/notifier.h>
  18#include <linux/hardirq.h>
  19#include <linux/kprobes.h>
  20#include <linux/module.h>
  21#include <linux/kdebug.h>
  22#include <linux/sched.h>
  23#include <linux/uaccess.h>
  24#include <linux/slab.h>
  25#include <linux/cpu.h>
  26#include <linux/bitops.h>
  27#include <linux/device.h>
  28
  29#include <asm/apic.h>
  30#include <asm/stacktrace.h>
  31#include <asm/nmi.h>
  32#include <asm/smp.h>
  33#include <asm/alternative.h>
  34#include <asm/timer.h>
 
 
  35
  36#include "perf_event.h"
  37
  38#if 0
  39#undef wrmsrl
  40#define wrmsrl(msr, val) 					\
  41do {								\
  42	trace_printk("wrmsrl(%lx, %lx)\n", (unsigned long)(msr),\
  43			(unsigned long)(val));			\
  44	native_write_msr((msr), (u32)((u64)(val)), 		\
  45			(u32)((u64)(val) >> 32));		\
  46} while (0)
  47#endif
  48
  49struct x86_pmu x86_pmu __read_mostly;
  50
  51DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  52	.enabled = 1,
  53};
  54
  55u64 __read_mostly hw_cache_event_ids
  56				[PERF_COUNT_HW_CACHE_MAX]
  57				[PERF_COUNT_HW_CACHE_OP_MAX]
  58				[PERF_COUNT_HW_CACHE_RESULT_MAX];
  59u64 __read_mostly hw_cache_extra_regs
  60				[PERF_COUNT_HW_CACHE_MAX]
  61				[PERF_COUNT_HW_CACHE_OP_MAX]
  62				[PERF_COUNT_HW_CACHE_RESULT_MAX];
  63
  64/*
  65 * Propagate event elapsed time into the generic event.
  66 * Can only be executed on the CPU where the event is active.
  67 * Returns the delta events processed.
  68 */
  69u64 x86_perf_event_update(struct perf_event *event)
  70{
  71	struct hw_perf_event *hwc = &event->hw;
  72	int shift = 64 - x86_pmu.cntval_bits;
  73	u64 prev_raw_count, new_raw_count;
  74	int idx = hwc->idx;
  75	s64 delta;
  76
  77	if (idx == X86_PMC_IDX_FIXED_BTS)
  78		return 0;
  79
  80	/*
  81	 * Careful: an NMI might modify the previous event value.
  82	 *
  83	 * Our tactic to handle this is to first atomically read and
  84	 * exchange a new raw count - then add that new-prev delta
  85	 * count to the generic event atomically:
  86	 */
  87again:
  88	prev_raw_count = local64_read(&hwc->prev_count);
  89	rdmsrl(hwc->event_base, new_raw_count);
  90
  91	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  92					new_raw_count) != prev_raw_count)
  93		goto again;
  94
  95	/*
  96	 * Now we have the new raw value and have updated the prev
  97	 * timestamp already. We can now calculate the elapsed delta
  98	 * (event-)time and add that to the generic event.
  99	 *
 100	 * Careful, not all hw sign-extends above the physical width
 101	 * of the count.
 102	 */
 103	delta = (new_raw_count << shift) - (prev_raw_count << shift);
 104	delta >>= shift;
 105
 106	local64_add(delta, &event->count);
 107	local64_sub(delta, &hwc->period_left);
 108
 109	return new_raw_count;
 110}
 111
 112/*
 113 * Find and validate any extra registers to set up.
 114 */
 115static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
 116{
 117	struct hw_perf_event_extra *reg;
 118	struct extra_reg *er;
 119
 120	reg = &event->hw.extra_reg;
 121
 122	if (!x86_pmu.extra_regs)
 123		return 0;
 124
 125	for (er = x86_pmu.extra_regs; er->msr; er++) {
 126		if (er->event != (config & er->config_mask))
 127			continue;
 128		if (event->attr.config1 & ~er->valid_mask)
 129			return -EINVAL;
 130
 131		reg->idx = er->idx;
 132		reg->config = event->attr.config1;
 133		reg->reg = er->msr;
 134		break;
 135	}
 136	return 0;
 137}
 138
 139static atomic_t active_events;
 140static DEFINE_MUTEX(pmc_reserve_mutex);
 141
 142#ifdef CONFIG_X86_LOCAL_APIC
 143
 144static bool reserve_pmc_hardware(void)
 145{
 146	int i;
 147
 148	for (i = 0; i < x86_pmu.num_counters; i++) {
 149		if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
 150			goto perfctr_fail;
 151	}
 152
 153	for (i = 0; i < x86_pmu.num_counters; i++) {
 154		if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
 155			goto eventsel_fail;
 156	}
 157
 158	return true;
 159
 160eventsel_fail:
 161	for (i--; i >= 0; i--)
 162		release_evntsel_nmi(x86_pmu_config_addr(i));
 163
 164	i = x86_pmu.num_counters;
 165
 166perfctr_fail:
 167	for (i--; i >= 0; i--)
 168		release_perfctr_nmi(x86_pmu_event_addr(i));
 169
 170	return false;
 171}
 172
 173static void release_pmc_hardware(void)
 174{
 175	int i;
 176
 177	for (i = 0; i < x86_pmu.num_counters; i++) {
 178		release_perfctr_nmi(x86_pmu_event_addr(i));
 179		release_evntsel_nmi(x86_pmu_config_addr(i));
 180	}
 181}
 182
 183#else
 184
 185static bool reserve_pmc_hardware(void) { return true; }
 186static void release_pmc_hardware(void) {}
 187
 188#endif
 189
 190static bool check_hw_exists(void)
 191{
 192	u64 val, val_new = 0;
 193	int i, reg, ret = 0;
 
 194
 195	/*
 196	 * Check to see if the BIOS enabled any of the counters, if so
 197	 * complain and bail.
 198	 */
 199	for (i = 0; i < x86_pmu.num_counters; i++) {
 200		reg = x86_pmu_config_addr(i);
 201		ret = rdmsrl_safe(reg, &val);
 202		if (ret)
 203			goto msr_fail;
 204		if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
 205			goto bios_fail;
 
 
 
 206	}
 207
 208	if (x86_pmu.num_counters_fixed) {
 209		reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 210		ret = rdmsrl_safe(reg, &val);
 211		if (ret)
 212			goto msr_fail;
 213		for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
 214			if (val & (0x03 << i*4))
 215				goto bios_fail;
 
 
 
 216		}
 217	}
 218
 219	/*
 220	 * Now write a value and read it back to see if it matches,
 221	 * this is needed to detect certain hardware emulators (qemu/kvm)
 222	 * that don't trap on the MSR access and always return 0s.
 223	 */
 224	val = 0xabcdUL;
 225	ret = checking_wrmsrl(x86_pmu_event_addr(0), val);
 226	ret |= rdmsrl_safe(x86_pmu_event_addr(0), &val_new);
 
 
 
 227	if (ret || val != val_new)
 228		goto msr_fail;
 229
 230	return true;
 231
 232bios_fail:
 233	/*
 234	 * We still allow the PMU driver to operate:
 235	 */
 236	printk(KERN_CONT "Broken BIOS detected, complain to your hardware vendor.\n");
 237	printk(KERN_ERR FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", reg, val);
 
 
 238
 239	return true;
 240
 241msr_fail:
 242	printk(KERN_CONT "Broken PMU hardware detected, using software events only.\n");
 
 243
 244	return false;
 245}
 246
 247static void hw_perf_event_destroy(struct perf_event *event)
 248{
 249	if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
 250		release_pmc_hardware();
 251		release_ds_buffers();
 252		mutex_unlock(&pmc_reserve_mutex);
 253	}
 254}
 255
 256static inline int x86_pmu_initialized(void)
 257{
 258	return x86_pmu.handle_irq != NULL;
 259}
 260
 261static inline int
 262set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
 263{
 264	struct perf_event_attr *attr = &event->attr;
 265	unsigned int cache_type, cache_op, cache_result;
 266	u64 config, val;
 267
 268	config = attr->config;
 269
 270	cache_type = (config >>  0) & 0xff;
 271	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 272		return -EINVAL;
 273
 274	cache_op = (config >>  8) & 0xff;
 275	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 276		return -EINVAL;
 277
 278	cache_result = (config >> 16) & 0xff;
 279	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 280		return -EINVAL;
 281
 282	val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 283
 284	if (val == 0)
 285		return -ENOENT;
 286
 287	if (val == -1)
 288		return -EINVAL;
 289
 290	hwc->config |= val;
 291	attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
 292	return x86_pmu_extra_regs(val, event);
 293}
 294
 295int x86_setup_perfctr(struct perf_event *event)
 296{
 297	struct perf_event_attr *attr = &event->attr;
 298	struct hw_perf_event *hwc = &event->hw;
 299	u64 config;
 300
 301	if (!is_sampling_event(event)) {
 302		hwc->sample_period = x86_pmu.max_period;
 303		hwc->last_period = hwc->sample_period;
 304		local64_set(&hwc->period_left, hwc->sample_period);
 305	} else {
 306		/*
 307		 * If we have a PMU initialized but no APIC
 308		 * interrupts, we cannot sample hardware
 309		 * events (user-space has to fall back and
 310		 * sample via a hrtimer based software event):
 311		 */
 312		if (!x86_pmu.apic)
 313			return -EOPNOTSUPP;
 314	}
 315
 316	if (attr->type == PERF_TYPE_RAW)
 317		return x86_pmu_extra_regs(event->attr.config, event);
 318
 319	if (attr->type == PERF_TYPE_HW_CACHE)
 320		return set_ext_hw_attr(hwc, event);
 321
 322	if (attr->config >= x86_pmu.max_events)
 323		return -EINVAL;
 324
 325	/*
 326	 * The generic map:
 327	 */
 328	config = x86_pmu.event_map(attr->config);
 329
 330	if (config == 0)
 331		return -ENOENT;
 332
 333	if (config == -1LL)
 334		return -EINVAL;
 335
 336	/*
 337	 * Branch tracing:
 338	 */
 339	if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
 340	    !attr->freq && hwc->sample_period == 1) {
 341		/* BTS is not supported by this architecture. */
 342		if (!x86_pmu.bts_active)
 343			return -EOPNOTSUPP;
 344
 345		/* BTS is currently only allowed for user-mode. */
 346		if (!attr->exclude_kernel)
 347			return -EOPNOTSUPP;
 348	}
 349
 350	hwc->config |= config;
 351
 352	return 0;
 353}
 354
 355/*
 356 * check that branch_sample_type is compatible with
 357 * settings needed for precise_ip > 1 which implies
 358 * using the LBR to capture ALL taken branches at the
 359 * priv levels of the measurement
 360 */
 361static inline int precise_br_compat(struct perf_event *event)
 362{
 363	u64 m = event->attr.branch_sample_type;
 364	u64 b = 0;
 365
 366	/* must capture all branches */
 367	if (!(m & PERF_SAMPLE_BRANCH_ANY))
 368		return 0;
 369
 370	m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
 371
 372	if (!event->attr.exclude_user)
 373		b |= PERF_SAMPLE_BRANCH_USER;
 374
 375	if (!event->attr.exclude_kernel)
 376		b |= PERF_SAMPLE_BRANCH_KERNEL;
 377
 378	/*
 379	 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
 380	 */
 381
 382	return m == b;
 383}
 384
 385int x86_pmu_hw_config(struct perf_event *event)
 386{
 387	if (event->attr.precise_ip) {
 388		int precise = 0;
 389
 390		/* Support for constant skid */
 391		if (x86_pmu.pebs_active) {
 392			precise++;
 393
 394			/* Support for IP fixup */
 395			if (x86_pmu.lbr_nr)
 396				precise++;
 397		}
 398
 399		if (event->attr.precise_ip > precise)
 400			return -EOPNOTSUPP;
 401		/*
 402		 * check that PEBS LBR correction does not conflict with
 403		 * whatever the user is asking with attr->branch_sample_type
 404		 */
 405		if (event->attr.precise_ip > 1) {
 
 406			u64 *br_type = &event->attr.branch_sample_type;
 407
 408			if (has_branch_stack(event)) {
 409				if (!precise_br_compat(event))
 410					return -EOPNOTSUPP;
 411
 412				/* branch_sample_type is compatible */
 413
 414			} else {
 415				/*
 416				 * user did not specify  branch_sample_type
 417				 *
 418				 * For PEBS fixups, we capture all
 419				 * the branches at the priv level of the
 420				 * event.
 421				 */
 422				*br_type = PERF_SAMPLE_BRANCH_ANY;
 423
 424				if (!event->attr.exclude_user)
 425					*br_type |= PERF_SAMPLE_BRANCH_USER;
 426
 427				if (!event->attr.exclude_kernel)
 428					*br_type |= PERF_SAMPLE_BRANCH_KERNEL;
 429			}
 430		}
 431	}
 432
 433	/*
 434	 * Generate PMC IRQs:
 435	 * (keep 'enabled' bit clear for now)
 436	 */
 437	event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
 438
 439	/*
 440	 * Count user and OS events unless requested not to
 441	 */
 442	if (!event->attr.exclude_user)
 443		event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
 444	if (!event->attr.exclude_kernel)
 445		event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
 446
 447	if (event->attr.type == PERF_TYPE_RAW)
 448		event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
 449
 450	return x86_setup_perfctr(event);
 451}
 452
 453/*
 454 * Setup the hardware configuration for a given attr_type
 455 */
 456static int __x86_pmu_event_init(struct perf_event *event)
 457{
 458	int err;
 459
 460	if (!x86_pmu_initialized())
 461		return -ENODEV;
 462
 463	err = 0;
 464	if (!atomic_inc_not_zero(&active_events)) {
 465		mutex_lock(&pmc_reserve_mutex);
 466		if (atomic_read(&active_events) == 0) {
 467			if (!reserve_pmc_hardware())
 468				err = -EBUSY;
 469			else
 470				reserve_ds_buffers();
 471		}
 472		if (!err)
 473			atomic_inc(&active_events);
 474		mutex_unlock(&pmc_reserve_mutex);
 475	}
 476	if (err)
 477		return err;
 478
 479	event->destroy = hw_perf_event_destroy;
 480
 481	event->hw.idx = -1;
 482	event->hw.last_cpu = -1;
 483	event->hw.last_tag = ~0ULL;
 484
 485	/* mark unused */
 486	event->hw.extra_reg.idx = EXTRA_REG_NONE;
 487	event->hw.branch_reg.idx = EXTRA_REG_NONE;
 488
 489	return x86_pmu.hw_config(event);
 490}
 491
 492void x86_pmu_disable_all(void)
 493{
 494	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 495	int idx;
 496
 497	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 498		u64 val;
 499
 500		if (!test_bit(idx, cpuc->active_mask))
 501			continue;
 502		rdmsrl(x86_pmu_config_addr(idx), val);
 503		if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
 504			continue;
 505		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
 506		wrmsrl(x86_pmu_config_addr(idx), val);
 507	}
 508}
 509
 510static void x86_pmu_disable(struct pmu *pmu)
 511{
 512	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 513
 514	if (!x86_pmu_initialized())
 515		return;
 516
 517	if (!cpuc->enabled)
 518		return;
 519
 520	cpuc->n_added = 0;
 521	cpuc->enabled = 0;
 522	barrier();
 523
 524	x86_pmu.disable_all();
 525}
 526
 527void x86_pmu_enable_all(int added)
 528{
 529	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 530	int idx;
 531
 532	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 533		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
 534
 535		if (!test_bit(idx, cpuc->active_mask))
 536			continue;
 537
 538		__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
 539	}
 540}
 541
 542static struct pmu pmu;
 543
 544static inline int is_x86_event(struct perf_event *event)
 545{
 546	return event->pmu == &pmu;
 547}
 548
 549/*
 550 * Event scheduler state:
 551 *
 552 * Assign events iterating over all events and counters, beginning
 553 * with events with least weights first. Keep the current iterator
 554 * state in struct sched_state.
 555 */
 556struct sched_state {
 557	int	weight;
 558	int	event;		/* event index */
 559	int	counter;	/* counter index */
 560	int	unassigned;	/* number of events to be assigned left */
 561	unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 562};
 563
 564/* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
 565#define	SCHED_STATES_MAX	2
 566
 567struct perf_sched {
 568	int			max_weight;
 569	int			max_events;
 570	struct event_constraint	**constraints;
 571	struct sched_state	state;
 572	int			saved_states;
 573	struct sched_state	saved[SCHED_STATES_MAX];
 574};
 575
 576/*
 577 * Initialize interator that runs through all events and counters.
 578 */
 579static void perf_sched_init(struct perf_sched *sched, struct event_constraint **c,
 580			    int num, int wmin, int wmax)
 581{
 582	int idx;
 583
 584	memset(sched, 0, sizeof(*sched));
 585	sched->max_events	= num;
 586	sched->max_weight	= wmax;
 587	sched->constraints	= c;
 588
 589	for (idx = 0; idx < num; idx++) {
 590		if (c[idx]->weight == wmin)
 591			break;
 592	}
 593
 594	sched->state.event	= idx;		/* start with min weight */
 595	sched->state.weight	= wmin;
 596	sched->state.unassigned	= num;
 597}
 598
 599static void perf_sched_save_state(struct perf_sched *sched)
 600{
 601	if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
 602		return;
 603
 604	sched->saved[sched->saved_states] = sched->state;
 605	sched->saved_states++;
 606}
 607
 608static bool perf_sched_restore_state(struct perf_sched *sched)
 609{
 610	if (!sched->saved_states)
 611		return false;
 612
 613	sched->saved_states--;
 614	sched->state = sched->saved[sched->saved_states];
 615
 616	/* continue with next counter: */
 617	clear_bit(sched->state.counter++, sched->state.used);
 618
 619	return true;
 620}
 621
 622/*
 623 * Select a counter for the current event to schedule. Return true on
 624 * success.
 625 */
 626static bool __perf_sched_find_counter(struct perf_sched *sched)
 627{
 628	struct event_constraint *c;
 629	int idx;
 630
 631	if (!sched->state.unassigned)
 632		return false;
 633
 634	if (sched->state.event >= sched->max_events)
 635		return false;
 636
 637	c = sched->constraints[sched->state.event];
 638
 639	/* Prefer fixed purpose counters */
 640	if (x86_pmu.num_counters_fixed) {
 641		idx = X86_PMC_IDX_FIXED;
 642		for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
 643			if (!__test_and_set_bit(idx, sched->state.used))
 644				goto done;
 645		}
 646	}
 647	/* Grab the first unused counter starting with idx */
 648	idx = sched->state.counter;
 649	for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_FIXED) {
 650		if (!__test_and_set_bit(idx, sched->state.used))
 651			goto done;
 652	}
 653
 654	return false;
 655
 656done:
 657	sched->state.counter = idx;
 658
 659	if (c->overlap)
 660		perf_sched_save_state(sched);
 661
 662	return true;
 663}
 664
 665static bool perf_sched_find_counter(struct perf_sched *sched)
 666{
 667	while (!__perf_sched_find_counter(sched)) {
 668		if (!perf_sched_restore_state(sched))
 669			return false;
 670	}
 671
 672	return true;
 673}
 674
 675/*
 676 * Go through all unassigned events and find the next one to schedule.
 677 * Take events with the least weight first. Return true on success.
 678 */
 679static bool perf_sched_next_event(struct perf_sched *sched)
 680{
 681	struct event_constraint *c;
 682
 683	if (!sched->state.unassigned || !--sched->state.unassigned)
 684		return false;
 685
 686	do {
 687		/* next event */
 688		sched->state.event++;
 689		if (sched->state.event >= sched->max_events) {
 690			/* next weight */
 691			sched->state.event = 0;
 692			sched->state.weight++;
 693			if (sched->state.weight > sched->max_weight)
 694				return false;
 695		}
 696		c = sched->constraints[sched->state.event];
 697	} while (c->weight != sched->state.weight);
 698
 699	sched->state.counter = 0;	/* start with first counter */
 700
 701	return true;
 702}
 703
 704/*
 705 * Assign a counter for each event.
 706 */
 707static int perf_assign_events(struct event_constraint **constraints, int n,
 708			      int wmin, int wmax, int *assign)
 709{
 710	struct perf_sched sched;
 711
 712	perf_sched_init(&sched, constraints, n, wmin, wmax);
 713
 714	do {
 715		if (!perf_sched_find_counter(&sched))
 716			break;	/* failed */
 717		if (assign)
 718			assign[sched.state.event] = sched.state.counter;
 719	} while (perf_sched_next_event(&sched));
 720
 721	return sched.state.unassigned;
 722}
 723
 724int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
 725{
 726	struct event_constraint *c, *constraints[X86_PMC_IDX_MAX];
 727	unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 
 728	int i, wmin, wmax, num = 0;
 729	struct hw_perf_event *hwc;
 730
 731	bitmap_zero(used_mask, X86_PMC_IDX_MAX);
 732
 733	for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
 
 734		c = x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
 735		constraints[i] = c;
 
 736		wmin = min(wmin, c->weight);
 737		wmax = max(wmax, c->weight);
 738	}
 739
 740	/*
 741	 * fastpath, try to reuse previous register
 742	 */
 743	for (i = 0; i < n; i++) {
 744		hwc = &cpuc->event_list[i]->hw;
 745		c = constraints[i];
 746
 747		/* never assigned */
 748		if (hwc->idx == -1)
 749			break;
 750
 751		/* constraint still honored */
 752		if (!test_bit(hwc->idx, c->idxmsk))
 753			break;
 754
 755		/* not already used */
 756		if (test_bit(hwc->idx, used_mask))
 757			break;
 758
 759		__set_bit(hwc->idx, used_mask);
 760		if (assign)
 761			assign[i] = hwc->idx;
 762	}
 763
 764	/* slow path */
 765	if (i != n)
 766		num = perf_assign_events(constraints, n, wmin, wmax, assign);
 
 767
 768	/*
 
 
 
 
 
 
 
 
 
 
 769	 * scheduling failed or is just a simulation,
 770	 * free resources if necessary
 771	 */
 772	if (!assign || num) {
 773		for (i = 0; i < n; i++) {
 
 
 
 
 
 
 
 
 774			if (x86_pmu.put_event_constraints)
 775				x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
 776		}
 777	}
 778	return num ? -EINVAL : 0;
 779}
 780
 781/*
 782 * dogrp: true if must collect siblings events (group)
 783 * returns total number of events and error code
 784 */
 785static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
 786{
 787	struct perf_event *event;
 788	int n, max_count;
 789
 790	max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
 791
 792	/* current number of events already accepted */
 793	n = cpuc->n_events;
 794
 795	if (is_x86_event(leader)) {
 796		if (n >= max_count)
 797			return -EINVAL;
 798		cpuc->event_list[n] = leader;
 799		n++;
 800	}
 801	if (!dogrp)
 802		return n;
 803
 804	list_for_each_entry(event, &leader->sibling_list, group_entry) {
 805		if (!is_x86_event(event) ||
 806		    event->state <= PERF_EVENT_STATE_OFF)
 807			continue;
 808
 809		if (n >= max_count)
 810			return -EINVAL;
 811
 812		cpuc->event_list[n] = event;
 813		n++;
 814	}
 815	return n;
 816}
 817
 818static inline void x86_assign_hw_event(struct perf_event *event,
 819				struct cpu_hw_events *cpuc, int i)
 820{
 821	struct hw_perf_event *hwc = &event->hw;
 822
 823	hwc->idx = cpuc->assign[i];
 824	hwc->last_cpu = smp_processor_id();
 825	hwc->last_tag = ++cpuc->tags[i];
 826
 827	if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
 828		hwc->config_base = 0;
 829		hwc->event_base	= 0;
 830	} else if (hwc->idx >= X86_PMC_IDX_FIXED) {
 831		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 832		hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - X86_PMC_IDX_FIXED);
 
 833	} else {
 834		hwc->config_base = x86_pmu_config_addr(hwc->idx);
 835		hwc->event_base  = x86_pmu_event_addr(hwc->idx);
 
 836	}
 837}
 838
 839static inline int match_prev_assignment(struct hw_perf_event *hwc,
 840					struct cpu_hw_events *cpuc,
 841					int i)
 842{
 843	return hwc->idx == cpuc->assign[i] &&
 844		hwc->last_cpu == smp_processor_id() &&
 845		hwc->last_tag == cpuc->tags[i];
 846}
 847
 848static void x86_pmu_start(struct perf_event *event, int flags);
 849
 850static void x86_pmu_enable(struct pmu *pmu)
 851{
 852	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 853	struct perf_event *event;
 854	struct hw_perf_event *hwc;
 855	int i, added = cpuc->n_added;
 856
 857	if (!x86_pmu_initialized())
 858		return;
 859
 860	if (cpuc->enabled)
 861		return;
 862
 863	if (cpuc->n_added) {
 864		int n_running = cpuc->n_events - cpuc->n_added;
 865		/*
 866		 * apply assignment obtained either from
 867		 * hw_perf_group_sched_in() or x86_pmu_enable()
 868		 *
 869		 * step1: save events moving to new counters
 870		 * step2: reprogram moved events into new counters
 871		 */
 872		for (i = 0; i < n_running; i++) {
 873			event = cpuc->event_list[i];
 874			hwc = &event->hw;
 875
 876			/*
 877			 * we can avoid reprogramming counter if:
 878			 * - assigned same counter as last time
 879			 * - running on same CPU as last time
 880			 * - no other event has used the counter since
 881			 */
 882			if (hwc->idx == -1 ||
 883			    match_prev_assignment(hwc, cpuc, i))
 884				continue;
 885
 886			/*
 887			 * Ensure we don't accidentally enable a stopped
 888			 * counter simply because we rescheduled.
 889			 */
 890			if (hwc->state & PERF_HES_STOPPED)
 891				hwc->state |= PERF_HES_ARCH;
 892
 893			x86_pmu_stop(event, PERF_EF_UPDATE);
 894		}
 895
 
 
 
 896		for (i = 0; i < cpuc->n_events; i++) {
 897			event = cpuc->event_list[i];
 898			hwc = &event->hw;
 899
 900			if (!match_prev_assignment(hwc, cpuc, i))
 901				x86_assign_hw_event(event, cpuc, i);
 902			else if (i < n_running)
 903				continue;
 904
 905			if (hwc->state & PERF_HES_ARCH)
 906				continue;
 907
 908			x86_pmu_start(event, PERF_EF_RELOAD);
 909		}
 910		cpuc->n_added = 0;
 911		perf_events_lapic_init();
 912	}
 913
 914	cpuc->enabled = 1;
 915	barrier();
 916
 917	x86_pmu.enable_all(added);
 918}
 919
 920static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
 921
 922/*
 923 * Set the next IRQ period, based on the hwc->period_left value.
 924 * To be called with the event disabled in hw:
 925 */
 926int x86_perf_event_set_period(struct perf_event *event)
 927{
 928	struct hw_perf_event *hwc = &event->hw;
 929	s64 left = local64_read(&hwc->period_left);
 930	s64 period = hwc->sample_period;
 931	int ret = 0, idx = hwc->idx;
 932
 933	if (idx == X86_PMC_IDX_FIXED_BTS)
 934		return 0;
 935
 936	/*
 937	 * If we are way outside a reasonable range then just skip forward:
 938	 */
 939	if (unlikely(left <= -period)) {
 940		left = period;
 941		local64_set(&hwc->period_left, left);
 942		hwc->last_period = period;
 943		ret = 1;
 944	}
 945
 946	if (unlikely(left <= 0)) {
 947		left += period;
 948		local64_set(&hwc->period_left, left);
 949		hwc->last_period = period;
 950		ret = 1;
 951	}
 952	/*
 953	 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
 954	 */
 955	if (unlikely(left < 2))
 956		left = 2;
 957
 958	if (left > x86_pmu.max_period)
 959		left = x86_pmu.max_period;
 960
 961	per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
 962
 963	/*
 964	 * The hw event starts counting from this event offset,
 965	 * mark it to be able to extra future deltas:
 966	 */
 967	local64_set(&hwc->prev_count, (u64)-left);
 968
 969	wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
 970
 971	/*
 972	 * Due to erratum on certan cpu we need
 973	 * a second write to be sure the register
 974	 * is updated properly
 975	 */
 976	if (x86_pmu.perfctr_second_write) {
 977		wrmsrl(hwc->event_base,
 978			(u64)(-left) & x86_pmu.cntval_mask);
 979	}
 980
 981	perf_event_update_userpage(event);
 982
 983	return ret;
 984}
 985
 986void x86_pmu_enable_event(struct perf_event *event)
 987{
 988	if (__this_cpu_read(cpu_hw_events.enabled))
 989		__x86_pmu_enable_event(&event->hw,
 990				       ARCH_PERFMON_EVENTSEL_ENABLE);
 991}
 992
 993/*
 994 * Add a single event to the PMU.
 995 *
 996 * The event is added to the group of enabled events
 997 * but only if it can be scehduled with existing events.
 998 */
 999static int x86_pmu_add(struct perf_event *event, int flags)
1000{
1001	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1002	struct hw_perf_event *hwc;
1003	int assign[X86_PMC_IDX_MAX];
1004	int n, n0, ret;
1005
1006	hwc = &event->hw;
1007
1008	perf_pmu_disable(event->pmu);
1009	n0 = cpuc->n_events;
1010	ret = n = collect_events(cpuc, event, false);
1011	if (ret < 0)
1012		goto out;
1013
1014	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1015	if (!(flags & PERF_EF_START))
1016		hwc->state |= PERF_HES_ARCH;
1017
1018	/*
1019	 * If group events scheduling transaction was started,
1020	 * skip the schedulability test here, it will be performed
1021	 * at commit time (->commit_txn) as a whole
1022	 */
1023	if (cpuc->group_flag & PERF_EVENT_TXN)
1024		goto done_collect;
1025
1026	ret = x86_pmu.schedule_events(cpuc, n, assign);
1027	if (ret)
1028		goto out;
1029	/*
1030	 * copy new assignment, now we know it is possible
1031	 * will be used by hw_perf_enable()
1032	 */
1033	memcpy(cpuc->assign, assign, n*sizeof(int));
1034
1035done_collect:
 
 
 
 
1036	cpuc->n_events = n;
1037	cpuc->n_added += n - n0;
1038	cpuc->n_txn += n - n0;
1039
1040	ret = 0;
1041out:
1042	perf_pmu_enable(event->pmu);
1043	return ret;
1044}
1045
1046static void x86_pmu_start(struct perf_event *event, int flags)
1047{
1048	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1049	int idx = event->hw.idx;
1050
1051	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1052		return;
1053
1054	if (WARN_ON_ONCE(idx == -1))
1055		return;
1056
1057	if (flags & PERF_EF_RELOAD) {
1058		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1059		x86_perf_event_set_period(event);
1060	}
1061
1062	event->hw.state = 0;
1063
1064	cpuc->events[idx] = event;
1065	__set_bit(idx, cpuc->active_mask);
1066	__set_bit(idx, cpuc->running);
1067	x86_pmu.enable(event);
1068	perf_event_update_userpage(event);
1069}
1070
1071void perf_event_print_debug(void)
1072{
1073	u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1074	u64 pebs;
1075	struct cpu_hw_events *cpuc;
1076	unsigned long flags;
1077	int cpu, idx;
1078
1079	if (!x86_pmu.num_counters)
1080		return;
1081
1082	local_irq_save(flags);
1083
1084	cpu = smp_processor_id();
1085	cpuc = &per_cpu(cpu_hw_events, cpu);
1086
1087	if (x86_pmu.version >= 2) {
1088		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1089		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1090		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1091		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1092		rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1093
1094		pr_info("\n");
1095		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1096		pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1097		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1098		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1099		pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
1100	}
1101	pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
1102
1103	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1104		rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1105		rdmsrl(x86_pmu_event_addr(idx), pmc_count);
1106
1107		prev_left = per_cpu(pmc_prev_left[idx], cpu);
1108
1109		pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1110			cpu, idx, pmc_ctrl);
1111		pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1112			cpu, idx, pmc_count);
1113		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1114			cpu, idx, prev_left);
1115	}
1116	for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1117		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1118
1119		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1120			cpu, idx, pmc_count);
1121	}
1122	local_irq_restore(flags);
1123}
1124
1125void x86_pmu_stop(struct perf_event *event, int flags)
1126{
1127	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1128	struct hw_perf_event *hwc = &event->hw;
1129
1130	if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
1131		x86_pmu.disable(event);
1132		cpuc->events[hwc->idx] = NULL;
1133		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1134		hwc->state |= PERF_HES_STOPPED;
1135	}
1136
1137	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1138		/*
1139		 * Drain the remaining delta count out of a event
1140		 * that we are disabling:
1141		 */
1142		x86_perf_event_update(event);
1143		hwc->state |= PERF_HES_UPTODATE;
1144	}
1145}
1146
1147static void x86_pmu_del(struct perf_event *event, int flags)
1148{
1149	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1150	int i;
1151
1152	/*
 
 
 
 
 
1153	 * If we're called during a txn, we don't need to do anything.
1154	 * The events never got scheduled and ->cancel_txn will truncate
1155	 * the event_list.
 
 
 
1156	 */
1157	if (cpuc->group_flag & PERF_EVENT_TXN)
1158		return;
1159
 
 
 
1160	x86_pmu_stop(event, PERF_EF_UPDATE);
1161
1162	for (i = 0; i < cpuc->n_events; i++) {
1163		if (event == cpuc->event_list[i]) {
 
 
1164
1165			if (x86_pmu.put_event_constraints)
1166				x86_pmu.put_event_constraints(cpuc, event);
1167
1168			while (++i < cpuc->n_events)
1169				cpuc->event_list[i-1] = cpuc->event_list[i];
 
 
 
 
 
 
 
 
 
1170
1171			--cpuc->n_events;
1172			break;
1173		}
1174	}
1175	perf_event_update_userpage(event);
1176}
1177
1178int x86_pmu_handle_irq(struct pt_regs *regs)
1179{
1180	struct perf_sample_data data;
1181	struct cpu_hw_events *cpuc;
1182	struct perf_event *event;
1183	int idx, handled = 0;
1184	u64 val;
1185
1186	cpuc = &__get_cpu_var(cpu_hw_events);
1187
1188	/*
1189	 * Some chipsets need to unmask the LVTPC in a particular spot
1190	 * inside the nmi handler.  As a result, the unmasking was pushed
1191	 * into all the nmi handlers.
1192	 *
1193	 * This generic handler doesn't seem to have any issues where the
1194	 * unmasking occurs so it was left at the top.
1195	 */
1196	apic_write(APIC_LVTPC, APIC_DM_NMI);
1197
1198	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1199		if (!test_bit(idx, cpuc->active_mask)) {
1200			/*
1201			 * Though we deactivated the counter some cpus
1202			 * might still deliver spurious interrupts still
1203			 * in flight. Catch them:
1204			 */
1205			if (__test_and_clear_bit(idx, cpuc->running))
1206				handled++;
1207			continue;
1208		}
1209
1210		event = cpuc->events[idx];
1211
1212		val = x86_perf_event_update(event);
1213		if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1214			continue;
1215
1216		/*
1217		 * event overflow
1218		 */
1219		handled++;
1220		perf_sample_data_init(&data, 0, event->hw.last_period);
1221
1222		if (!x86_perf_event_set_period(event))
1223			continue;
1224
1225		if (perf_event_overflow(event, &data, regs))
1226			x86_pmu_stop(event, 0);
1227	}
1228
1229	if (handled)
1230		inc_irq_stat(apic_perf_irqs);
1231
1232	return handled;
1233}
1234
1235void perf_events_lapic_init(void)
1236{
1237	if (!x86_pmu.apic || !x86_pmu_initialized())
1238		return;
1239
1240	/*
1241	 * Always use NMI for PMU
1242	 */
1243	apic_write(APIC_LVTPC, APIC_DM_NMI);
1244}
1245
1246static int __kprobes
1247perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1248{
 
 
 
 
1249	if (!atomic_read(&active_events))
1250		return NMI_DONE;
1251
1252	return x86_pmu.handle_irq(regs);
 
 
 
 
 
 
1253}
1254
1255struct event_constraint emptyconstraint;
1256struct event_constraint unconstrained;
1257
1258static int __cpuinit
1259x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
1260{
1261	unsigned int cpu = (long)hcpu;
1262	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1263	int ret = NOTIFY_OK;
1264
1265	switch (action & ~CPU_TASKS_FROZEN) {
1266	case CPU_UP_PREPARE:
1267		cpuc->kfree_on_online = NULL;
1268		if (x86_pmu.cpu_prepare)
1269			ret = x86_pmu.cpu_prepare(cpu);
1270		break;
1271
1272	case CPU_STARTING:
1273		if (x86_pmu.attr_rdpmc)
1274			set_in_cr4(X86_CR4_PCE);
1275		if (x86_pmu.cpu_starting)
1276			x86_pmu.cpu_starting(cpu);
1277		break;
1278
1279	case CPU_ONLINE:
1280		kfree(cpuc->kfree_on_online);
1281		break;
1282
1283	case CPU_DYING:
1284		if (x86_pmu.cpu_dying)
1285			x86_pmu.cpu_dying(cpu);
1286		break;
1287
1288	case CPU_UP_CANCELED:
1289	case CPU_DEAD:
1290		if (x86_pmu.cpu_dead)
1291			x86_pmu.cpu_dead(cpu);
1292		break;
1293
1294	default:
1295		break;
1296	}
1297
1298	return ret;
1299}
1300
1301static void __init pmu_check_apic(void)
1302{
1303	if (cpu_has_apic)
1304		return;
1305
1306	x86_pmu.apic = 0;
1307	pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1308	pr_info("no hardware sampling interrupt available.\n");
1309}
1310
1311static struct attribute_group x86_pmu_format_group = {
1312	.name = "format",
1313	.attrs = NULL,
1314};
1315
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1316static int __init init_hw_perf_events(void)
1317{
1318	struct x86_pmu_quirk *quirk;
1319	struct event_constraint *c;
1320	int err;
1321
1322	pr_info("Performance Events: ");
1323
1324	switch (boot_cpu_data.x86_vendor) {
1325	case X86_VENDOR_INTEL:
1326		err = intel_pmu_init();
1327		break;
1328	case X86_VENDOR_AMD:
1329		err = amd_pmu_init();
1330		break;
1331	default:
1332		return 0;
1333	}
1334	if (err != 0) {
1335		pr_cont("no PMU driver, software events only.\n");
1336		return 0;
1337	}
1338
1339	pmu_check_apic();
1340
1341	/* sanity check that the hardware exists or is emulated */
1342	if (!check_hw_exists())
1343		return 0;
1344
1345	pr_cont("%s PMU driver.\n", x86_pmu.name);
1346
 
 
1347	for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1348		quirk->func();
1349
1350	if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1351		WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
1352		     x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
1353		x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
1354	}
1355	x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
1356
1357	if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1358		WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
1359		     x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
1360		x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1361	}
1362
1363	x86_pmu.intel_ctrl |=
1364		((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1365
1366	perf_events_lapic_init();
1367	register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1368
1369	unconstrained = (struct event_constraint)
1370		__EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1371				   0, x86_pmu.num_counters, 0);
1372
1373	if (x86_pmu.event_constraints) {
1374		/*
1375		 * event on fixed counter2 (REF_CYCLES) only works on this
1376		 * counter, so do not extend mask to generic counters
1377		 */
1378		for_each_event_constraint(c, x86_pmu.event_constraints) {
1379			if (c->cmask != X86_RAW_EVENT_MASK
1380			    || c->idxmsk64 == X86_PMC_MSK_FIXED_REF_CYCLES) {
1381				continue;
1382			}
1383
1384			c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
1385			c->weight += x86_pmu.num_counters;
1386		}
1387	}
1388
1389	x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1390	x86_pmu_format_group.attrs = x86_pmu.format_attrs;
 
 
 
 
 
 
 
 
 
 
1391
1392	pr_info("... version:                %d\n",     x86_pmu.version);
1393	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
1394	pr_info("... generic registers:      %d\n",     x86_pmu.num_counters);
1395	pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
1396	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
1397	pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
1398	pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
1399
1400	perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1401	perf_cpu_notifier(x86_pmu_notifier);
1402
1403	return 0;
1404}
1405early_initcall(init_hw_perf_events);
1406
1407static inline void x86_pmu_read(struct perf_event *event)
1408{
1409	x86_perf_event_update(event);
1410}
1411
1412/*
1413 * Start group events scheduling transaction
1414 * Set the flag to make pmu::enable() not perform the
1415 * schedulability test, it will be performed at commit time
1416 */
1417static void x86_pmu_start_txn(struct pmu *pmu)
1418{
1419	perf_pmu_disable(pmu);
1420	__this_cpu_or(cpu_hw_events.group_flag, PERF_EVENT_TXN);
1421	__this_cpu_write(cpu_hw_events.n_txn, 0);
1422}
1423
1424/*
1425 * Stop group events scheduling transaction
1426 * Clear the flag and pmu::enable() will perform the
1427 * schedulability test.
1428 */
1429static void x86_pmu_cancel_txn(struct pmu *pmu)
1430{
1431	__this_cpu_and(cpu_hw_events.group_flag, ~PERF_EVENT_TXN);
1432	/*
1433	 * Truncate the collected events.
 
1434	 */
1435	__this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
1436	__this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
1437	perf_pmu_enable(pmu);
1438}
1439
1440/*
1441 * Commit group events scheduling transaction
1442 * Perform the group schedulability test as a whole
1443 * Return 0 if success
 
 
1444 */
1445static int x86_pmu_commit_txn(struct pmu *pmu)
1446{
1447	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1448	int assign[X86_PMC_IDX_MAX];
1449	int n, ret;
1450
1451	n = cpuc->n_events;
1452
1453	if (!x86_pmu_initialized())
1454		return -EAGAIN;
1455
1456	ret = x86_pmu.schedule_events(cpuc, n, assign);
1457	if (ret)
1458		return ret;
1459
1460	/*
1461	 * copy new assignment, now we know it is possible
1462	 * will be used by hw_perf_enable()
1463	 */
1464	memcpy(cpuc->assign, assign, n*sizeof(int));
1465
1466	cpuc->group_flag &= ~PERF_EVENT_TXN;
1467	perf_pmu_enable(pmu);
1468	return 0;
1469}
1470/*
1471 * a fake_cpuc is used to validate event groups. Due to
1472 * the extra reg logic, we need to also allocate a fake
1473 * per_core and per_cpu structure. Otherwise, group events
1474 * using extra reg may conflict without the kernel being
1475 * able to catch this when the last event gets added to
1476 * the group.
1477 */
1478static void free_fake_cpuc(struct cpu_hw_events *cpuc)
1479{
1480	kfree(cpuc->shared_regs);
1481	kfree(cpuc);
1482}
1483
1484static struct cpu_hw_events *allocate_fake_cpuc(void)
1485{
1486	struct cpu_hw_events *cpuc;
1487	int cpu = raw_smp_processor_id();
1488
1489	cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
1490	if (!cpuc)
1491		return ERR_PTR(-ENOMEM);
1492
1493	/* only needed, if we have extra_regs */
1494	if (x86_pmu.extra_regs) {
1495		cpuc->shared_regs = allocate_shared_regs(cpu);
1496		if (!cpuc->shared_regs)
1497			goto error;
1498	}
1499	cpuc->is_fake = 1;
1500	return cpuc;
1501error:
1502	free_fake_cpuc(cpuc);
1503	return ERR_PTR(-ENOMEM);
1504}
1505
1506/*
1507 * validate that we can schedule this event
1508 */
1509static int validate_event(struct perf_event *event)
1510{
1511	struct cpu_hw_events *fake_cpuc;
1512	struct event_constraint *c;
1513	int ret = 0;
1514
1515	fake_cpuc = allocate_fake_cpuc();
1516	if (IS_ERR(fake_cpuc))
1517		return PTR_ERR(fake_cpuc);
1518
1519	c = x86_pmu.get_event_constraints(fake_cpuc, event);
1520
1521	if (!c || !c->weight)
1522		ret = -EINVAL;
1523
1524	if (x86_pmu.put_event_constraints)
1525		x86_pmu.put_event_constraints(fake_cpuc, event);
1526
1527	free_fake_cpuc(fake_cpuc);
1528
1529	return ret;
1530}
1531
1532/*
1533 * validate a single event group
1534 *
1535 * validation include:
1536 *	- check events are compatible which each other
1537 *	- events do not compete for the same counter
1538 *	- number of events <= number of counters
1539 *
1540 * validation ensures the group can be loaded onto the
1541 * PMU if it was the only group available.
1542 */
1543static int validate_group(struct perf_event *event)
1544{
1545	struct perf_event *leader = event->group_leader;
1546	struct cpu_hw_events *fake_cpuc;
1547	int ret = -EINVAL, n;
1548
1549	fake_cpuc = allocate_fake_cpuc();
1550	if (IS_ERR(fake_cpuc))
1551		return PTR_ERR(fake_cpuc);
1552	/*
1553	 * the event is not yet connected with its
1554	 * siblings therefore we must first collect
1555	 * existing siblings, then add the new event
1556	 * before we can simulate the scheduling
1557	 */
1558	n = collect_events(fake_cpuc, leader, true);
1559	if (n < 0)
1560		goto out;
1561
1562	fake_cpuc->n_events = n;
1563	n = collect_events(fake_cpuc, event, false);
1564	if (n < 0)
1565		goto out;
1566
1567	fake_cpuc->n_events = n;
1568
1569	ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
1570
1571out:
1572	free_fake_cpuc(fake_cpuc);
1573	return ret;
1574}
1575
1576static int x86_pmu_event_init(struct perf_event *event)
1577{
1578	struct pmu *tmp;
1579	int err;
1580
1581	switch (event->attr.type) {
1582	case PERF_TYPE_RAW:
1583	case PERF_TYPE_HARDWARE:
1584	case PERF_TYPE_HW_CACHE:
1585		break;
1586
1587	default:
1588		return -ENOENT;
1589	}
1590
1591	err = __x86_pmu_event_init(event);
1592	if (!err) {
1593		/*
1594		 * we temporarily connect event to its pmu
1595		 * such that validate_group() can classify
1596		 * it as an x86 event using is_x86_event()
1597		 */
1598		tmp = event->pmu;
1599		event->pmu = &pmu;
1600
1601		if (event->group_leader != event)
1602			err = validate_group(event);
1603		else
1604			err = validate_event(event);
1605
1606		event->pmu = tmp;
1607	}
1608	if (err) {
1609		if (event->destroy)
1610			event->destroy(event);
1611	}
1612
1613	return err;
1614}
1615
1616static int x86_pmu_event_idx(struct perf_event *event)
1617{
1618	int idx = event->hw.idx;
1619
1620	if (!x86_pmu.attr_rdpmc)
1621		return 0;
1622
1623	if (x86_pmu.num_counters_fixed && idx >= X86_PMC_IDX_FIXED) {
1624		idx -= X86_PMC_IDX_FIXED;
1625		idx |= 1 << 30;
1626	}
1627
1628	return idx + 1;
1629}
1630
1631static ssize_t get_attr_rdpmc(struct device *cdev,
1632			      struct device_attribute *attr,
1633			      char *buf)
1634{
1635	return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
1636}
1637
1638static void change_rdpmc(void *info)
1639{
1640	bool enable = !!(unsigned long)info;
1641
1642	if (enable)
1643		set_in_cr4(X86_CR4_PCE);
1644	else
1645		clear_in_cr4(X86_CR4_PCE);
1646}
1647
1648static ssize_t set_attr_rdpmc(struct device *cdev,
1649			      struct device_attribute *attr,
1650			      const char *buf, size_t count)
1651{
1652	unsigned long val = simple_strtoul(buf, NULL, 0);
 
 
 
 
 
 
 
 
1653
1654	if (!!val != !!x86_pmu.attr_rdpmc) {
1655		x86_pmu.attr_rdpmc = !!val;
1656		smp_call_function(change_rdpmc, (void *)val, 1);
1657	}
1658
1659	return count;
1660}
1661
1662static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
1663
1664static struct attribute *x86_pmu_attrs[] = {
1665	&dev_attr_rdpmc.attr,
1666	NULL,
1667};
1668
1669static struct attribute_group x86_pmu_attr_group = {
1670	.attrs = x86_pmu_attrs,
1671};
1672
1673static const struct attribute_group *x86_pmu_attr_groups[] = {
1674	&x86_pmu_attr_group,
1675	&x86_pmu_format_group,
 
1676	NULL,
1677};
1678
1679static void x86_pmu_flush_branch_stack(void)
1680{
1681	if (x86_pmu.flush_branch_stack)
1682		x86_pmu.flush_branch_stack();
1683}
1684
 
 
 
 
 
 
 
1685static struct pmu pmu = {
1686	.pmu_enable		= x86_pmu_enable,
1687	.pmu_disable		= x86_pmu_disable,
1688
1689	.attr_groups	= x86_pmu_attr_groups,
1690
1691	.event_init	= x86_pmu_event_init,
1692
1693	.add			= x86_pmu_add,
1694	.del			= x86_pmu_del,
1695	.start			= x86_pmu_start,
1696	.stop			= x86_pmu_stop,
1697	.read			= x86_pmu_read,
1698
1699	.start_txn	= x86_pmu_start_txn,
1700	.cancel_txn	= x86_pmu_cancel_txn,
1701	.commit_txn	= x86_pmu_commit_txn,
1702
1703	.event_idx	= x86_pmu_event_idx,
1704	.flush_branch_stack	= x86_pmu_flush_branch_stack,
1705};
1706
1707void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
1708{
1709	userpg->cap_usr_time = 0;
1710	userpg->cap_usr_rdpmc = x86_pmu.attr_rdpmc;
 
 
 
1711	userpg->pmc_width = x86_pmu.cntval_bits;
1712
1713	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
1714		return;
1715
1716	if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1717		return;
 
 
 
 
 
 
 
1718
1719	userpg->cap_usr_time = 1;
1720	userpg->time_mult = this_cpu_read(cyc2ns);
1721	userpg->time_shift = CYC2NS_SCALE_FACTOR;
1722	userpg->time_offset = this_cpu_read(cyc2ns_offset) - now;
1723}
1724
1725/*
1726 * callchain support
1727 */
1728
1729static int backtrace_stack(void *data, char *name)
1730{
1731	return 0;
1732}
1733
1734static void backtrace_address(void *data, unsigned long addr, int reliable)
1735{
1736	struct perf_callchain_entry *entry = data;
1737
1738	perf_callchain_store(entry, addr);
1739}
1740
1741static const struct stacktrace_ops backtrace_ops = {
1742	.stack			= backtrace_stack,
1743	.address		= backtrace_address,
1744	.walk_stack		= print_context_stack_bp,
1745};
1746
1747void
1748perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
1749{
1750	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1751		/* TODO: We don't support guest os callchain now */
1752		return;
1753	}
1754
1755	perf_callchain_store(entry, regs->ip);
1756
1757	dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
1758}
1759
1760static inline int
1761valid_user_frame(const void __user *fp, unsigned long size)
1762{
1763	return (__range_not_ok(fp, size, TASK_SIZE) == 0);
1764}
1765
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1766#ifdef CONFIG_COMPAT
1767
1768#include <asm/compat.h>
1769
1770static inline int
1771perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
1772{
1773	/* 32-bit process in 64-bit kernel. */
 
1774	struct stack_frame_ia32 frame;
1775	const void __user *fp;
1776
1777	if (!test_thread_flag(TIF_IA32))
1778		return 0;
1779
1780	fp = compat_ptr(regs->bp);
 
 
 
1781	while (entry->nr < PERF_MAX_STACK_DEPTH) {
1782		unsigned long bytes;
1783		frame.next_frame     = 0;
1784		frame.return_address = 0;
1785
1786		bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
1787		if (bytes != sizeof(frame))
1788			break;
1789
1790		if (!valid_user_frame(fp, sizeof(frame)))
1791			break;
1792
1793		perf_callchain_store(entry, frame.return_address);
1794		fp = compat_ptr(frame.next_frame);
1795	}
1796	return 1;
1797}
1798#else
1799static inline int
1800perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
1801{
1802    return 0;
1803}
1804#endif
1805
1806void
1807perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
1808{
1809	struct stack_frame frame;
1810	const void __user *fp;
1811
1812	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1813		/* TODO: We don't support guest os callchain now */
1814		return;
1815	}
1816
 
 
 
 
 
 
1817	fp = (void __user *)regs->bp;
1818
1819	perf_callchain_store(entry, regs->ip);
1820
1821	if (!current->mm)
1822		return;
1823
1824	if (perf_callchain_user32(regs, entry))
1825		return;
1826
1827	while (entry->nr < PERF_MAX_STACK_DEPTH) {
1828		unsigned long bytes;
1829		frame.next_frame	     = NULL;
1830		frame.return_address = 0;
1831
1832		bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
1833		if (bytes != sizeof(frame))
1834			break;
1835
1836		if (!valid_user_frame(fp, sizeof(frame)))
1837			break;
1838
1839		perf_callchain_store(entry, frame.return_address);
1840		fp = frame.next_frame;
1841	}
1842}
1843
1844unsigned long perf_instruction_pointer(struct pt_regs *regs)
 
 
 
 
 
 
 
 
 
 
 
 
 
1845{
1846	unsigned long ip;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1847
 
 
1848	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
1849		ip = perf_guest_cbs->get_guest_ip();
1850	else
1851		ip = instruction_pointer(regs);
1852
1853	return ip;
1854}
1855
1856unsigned long perf_misc_flags(struct pt_regs *regs)
1857{
1858	int misc = 0;
1859
1860	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1861		if (perf_guest_cbs->is_user_mode())
1862			misc |= PERF_RECORD_MISC_GUEST_USER;
1863		else
1864			misc |= PERF_RECORD_MISC_GUEST_KERNEL;
1865	} else {
1866		if (user_mode(regs))
1867			misc |= PERF_RECORD_MISC_USER;
1868		else
1869			misc |= PERF_RECORD_MISC_KERNEL;
1870	}
1871
1872	if (regs->flags & PERF_EFLAGS_EXACT)
1873		misc |= PERF_RECORD_MISC_EXACT_IP;
1874
1875	return misc;
1876}
1877
1878void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
1879{
1880	cap->version		= x86_pmu.version;
1881	cap->num_counters_gp	= x86_pmu.num_counters;
1882	cap->num_counters_fixed	= x86_pmu.num_counters_fixed;
1883	cap->bit_width_gp	= x86_pmu.cntval_bits;
1884	cap->bit_width_fixed	= x86_pmu.cntval_bits;
1885	cap->events_mask	= (unsigned int)x86_pmu.events_maskl;
1886	cap->events_mask_len	= x86_pmu.events_mask_len;
1887}
1888EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);