1/*
   2 * Performance events core code:
   3 *
   4 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
   6 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   7 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   8 *
   9 * For licensing details see kernel-base/COPYING
  10 */
  11
  12#include <linux/fs.h>
  13#include <linux/mm.h>
  14#include <linux/cpu.h>
  15#include <linux/smp.h>
  16#include <linux/idr.h>
  17#include <linux/file.h>
  18#include <linux/poll.h>
  19#include <linux/slab.h>
  20#include <linux/hash.h>
 
  21#include <linux/sysfs.h>
  22#include <linux/dcache.h>
  23#include <linux/percpu.h>
  24#include <linux/ptrace.h>
  25#include <linux/reboot.h>
  26#include <linux/vmstat.h>
  27#include <linux/device.h>
  28#include <linux/export.h>
  29#include <linux/vmalloc.h>
  30#include <linux/hardirq.h>
  31#include <linux/rculist.h>
  32#include <linux/uaccess.h>
  33#include <linux/syscalls.h>
  34#include <linux/anon_inodes.h>
  35#include <linux/kernel_stat.h>
 
  36#include <linux/perf_event.h>
  37#include <linux/ftrace_event.h>
  38#include <linux/hw_breakpoint.h>
 
 
 
 
 
 
 
 
 
 
 
 
  39
  40#include "internal.h"
  41
  42#include <asm/irq_regs.h>
  43
 
 
  44struct remote_function_call {
  45	struct task_struct	*p;
  46	int			(*func)(void *info);
  47	void			*info;
  48	int			ret;
  49};
  50
  51static void remote_function(void *data)
  52{
  53	struct remote_function_call *tfc = data;
  54	struct task_struct *p = tfc->p;
  55
  56	if (p) {
  57		tfc->ret = -EAGAIN;
  58		if (task_cpu(p) != smp_processor_id() || !task_curr(p))
 
 
 
 
 
 
 
 
 
  59			return;
  60	}
  61
  62	tfc->ret = tfc->func(tfc->info);
  63}
  64
  65/**
  66 * task_function_call - call a function on the cpu on which a task runs
  67 * @p:		the task to evaluate
  68 * @func:	the function to be called
  69 * @info:	the function call argument
  70 *
  71 * Calls the function @func when the task is currently running. This might
  72 * be on the current CPU, which just calls the function directly
  73 *
  74 * returns: @func return value, or
  75 *	    -ESRCH  - when the process isn't running
  76 *	    -EAGAIN - when the process moved away
  77 */
  78static int
  79task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
  80{
  81	struct remote_function_call data = {
  82		.p	= p,
  83		.func	= func,
  84		.info	= info,
  85		.ret	= -ESRCH, /* No such (running) process */
  86	};
 
  87
  88	if (task_curr(p))
  89		smp_call_function_single(task_cpu(p), remote_function, &data, 1);
 
 
 
  90
  91	return data.ret;
  92}
  93
  94/**
  95 * cpu_function_call - call a function on the cpu
  96 * @func:	the function to be called
  97 * @info:	the function call argument
  98 *
  99 * Calls the function @func on the remote cpu.
 100 *
 101 * returns: @func return value or -ENXIO when the cpu is offline
 102 */
 103static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
 104{
 105	struct remote_function_call data = {
 106		.p	= NULL,
 107		.func	= func,
 108		.info	= info,
 109		.ret	= -ENXIO, /* No such CPU */
 110	};
 111
 112	smp_call_function_single(cpu, remote_function, &data, 1);
 113
 114	return data.ret;
 115}
 116
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 117#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
 118		       PERF_FLAG_FD_OUTPUT  |\
 119		       PERF_FLAG_PID_CGROUP)
 
 120
 121/*
 122 * branch priv levels that need permission checks
 123 */
 124#define PERF_SAMPLE_BRANCH_PERM_PLM \
 125	(PERF_SAMPLE_BRANCH_KERNEL |\
 126	 PERF_SAMPLE_BRANCH_HV)
 127
 128enum event_type_t {
 129	EVENT_FLEXIBLE = 0x1,
 130	EVENT_PINNED = 0x2,
 
 
 
 131	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
 132};
 133
 134/*
 135 * perf_sched_events : >0 events exist
 136 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
 137 */
 138struct static_key_deferred perf_sched_events __read_mostly;
 
 
 
 
 
 
 139static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
 140static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
 
 141
 142static atomic_t nr_mmap_events __read_mostly;
 143static atomic_t nr_comm_events __read_mostly;
 
 144static atomic_t nr_task_events __read_mostly;
 
 
 
 
 145
 146static LIST_HEAD(pmus);
 147static DEFINE_MUTEX(pmus_lock);
 148static struct srcu_struct pmus_srcu;
 
 149
 150/*
 151 * perf event paranoia level:
 152 *  -1 - not paranoid at all
 153 *   0 - disallow raw tracepoint access for unpriv
 154 *   1 - disallow cpu events for unpriv
 155 *   2 - disallow kernel profiling for unpriv
 156 */
 157int sysctl_perf_event_paranoid __read_mostly = 1;
 158
 159/* Minimum for 512 kiB + 1 user control page */
 160int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
 161
 162/*
 163 * max perf event sample rate
 164 */
 165#define DEFAULT_MAX_SAMPLE_RATE 100000
 166int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
 167static int max_samples_per_tick __read_mostly =
 168	DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 169
 170int perf_proc_update_handler(struct ctl_table *table, int write,
 171		void __user *buffer, size_t *lenp,
 172		loff_t *ppos)
 173{
 174	int ret = proc_dointvec(table, write, buffer, lenp, ppos);
 
 
 
 
 
 
 175
 
 176	if (ret || !write)
 177		return ret;
 178
 179	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 180
 181	return 0;
 182}
 183
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 184static atomic64_t perf_event_id;
 185
 186static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
 187			      enum event_type_t event_type);
 188
 189static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
 190			     enum event_type_t event_type,
 191			     struct task_struct *task);
 192
 193static void update_context_time(struct perf_event_context *ctx);
 194static u64 perf_event_time(struct perf_event *event);
 195
 196static void ring_buffer_attach(struct perf_event *event,
 197			       struct ring_buffer *rb);
 198
 199void __weak perf_event_print_debug(void)	{ }
 200
 201extern __weak const char *perf_pmu_name(void)
 202{
 203	return "pmu";
 204}
 205
 206static inline u64 perf_clock(void)
 207{
 208	return local_clock();
 209}
 210
 211static inline struct perf_cpu_context *
 212__get_cpu_context(struct perf_event_context *ctx)
 213{
 214	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
 215}
 216
 217static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
 218			  struct perf_event_context *ctx)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 219{
 220	raw_spin_lock(&cpuctx->ctx.lock);
 221	if (ctx)
 222		raw_spin_lock(&ctx->lock);
 
 
 
 223}
 224
 225static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
 226			    struct perf_event_context *ctx)
 227{
 228	if (ctx)
 229		raw_spin_unlock(&ctx->lock);
 230	raw_spin_unlock(&cpuctx->ctx.lock);
 231}
 232
 233#ifdef CONFIG_CGROUP_PERF
 
 
 234
 235/*
 236 * Must ensure cgroup is pinned (css_get) before calling
 237 * this function. In other words, we cannot call this function
 238 * if there is no cgroup event for the current CPU context.
 239 */
 240static inline struct perf_cgroup *
 241perf_cgroup_from_task(struct task_struct *task)
 242{
 243	return container_of(task_subsys_state(task, perf_subsys_id),
 244			struct perf_cgroup, css);
 
 
 
 245}
 246
 247static inline bool
 248perf_cgroup_match(struct perf_event *event)
 249{
 250	struct perf_event_context *ctx = event->ctx;
 251	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
 252
 253	return !event->cgrp || event->cgrp == cpuctx->cgrp;
 
 254}
 255
 256static inline bool perf_tryget_cgroup(struct perf_event *event)
 
 257{
 258	return css_tryget(&event->cgrp->css);
 
 
 
 
 
 
 
 
 
 
 
 259}
 260
 261static inline void perf_put_cgroup(struct perf_event *event)
 
 
 
 262{
 263	css_put(&event->cgrp->css);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 264}
 265
 266static inline void perf_detach_cgroup(struct perf_event *event)
 267{
 268	perf_put_cgroup(event);
 269	event->cgrp = NULL;
 270}
 271
 272static inline int is_cgroup_event(struct perf_event *event)
 273{
 274	return event->cgrp != NULL;
 275}
 276
 277static inline u64 perf_cgroup_event_time(struct perf_event *event)
 278{
 279	struct perf_cgroup_info *t;
 280
 281	t = per_cpu_ptr(event->cgrp->info, event->cpu);
 282	return t->time;
 283}
 284
 285static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
 286{
 287	struct perf_cgroup_info *info;
 288	u64 now;
 289
 290	now = perf_clock();
 291
 292	info = this_cpu_ptr(cgrp->info);
 293
 294	info->time += now - info->timestamp;
 295	info->timestamp = now;
 296}
 297
 298static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
 299{
 300	struct perf_cgroup *cgrp_out = cpuctx->cgrp;
 301	if (cgrp_out)
 302		__update_cgrp_time(cgrp_out);
 
 
 
 
 
 
 303}
 304
 305static inline void update_cgrp_time_from_event(struct perf_event *event)
 306{
 307	struct perf_cgroup *cgrp;
 308
 309	/*
 310	 * ensure we access cgroup data only when needed and
 311	 * when we know the cgroup is pinned (css_get)
 312	 */
 313	if (!is_cgroup_event(event))
 314		return;
 315
 316	cgrp = perf_cgroup_from_task(current);
 317	/*
 318	 * Do not update time when cgroup is not active
 319	 */
 320	if (cgrp == event->cgrp)
 321		__update_cgrp_time(event->cgrp);
 322}
 323
 324static inline void
 325perf_cgroup_set_timestamp(struct task_struct *task,
 326			  struct perf_event_context *ctx)
 327{
 328	struct perf_cgroup *cgrp;
 329	struct perf_cgroup_info *info;
 
 330
 331	/*
 332	 * ctx->lock held by caller
 333	 * ensure we do not access cgroup data
 334	 * unless we have the cgroup pinned (css_get)
 335	 */
 336	if (!task || !ctx->nr_cgroups)
 337		return;
 338
 339	cgrp = perf_cgroup_from_task(task);
 340	info = this_cpu_ptr(cgrp->info);
 341	info->timestamp = ctx->timestamp;
 
 
 
 
 342}
 343
 
 
 344#define PERF_CGROUP_SWOUT	0x1 /* cgroup switch out every event */
 345#define PERF_CGROUP_SWIN	0x2 /* cgroup switch in events based on task */
 346
 347/*
 348 * reschedule events based on the cgroup constraint of task.
 349 *
 350 * mode SWOUT : schedule out everything
 351 * mode SWIN : schedule in based on cgroup for next
 352 */
 353void perf_cgroup_switch(struct task_struct *task, int mode)
 354{
 355	struct perf_cpu_context *cpuctx;
 356	struct pmu *pmu;
 357	unsigned long flags;
 358
 359	/*
 360	 * disable interrupts to avoid geting nr_cgroup
 361	 * changes via __perf_event_disable(). Also
 362	 * avoids preemption.
 363	 */
 364	local_irq_save(flags);
 365
 366	/*
 367	 * we reschedule only in the presence of cgroup
 368	 * constrained events.
 369	 */
 370	rcu_read_lock();
 371
 372	list_for_each_entry_rcu(pmu, &pmus, entry) {
 373		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
 374
 375		/*
 376		 * perf_cgroup_events says at least one
 377		 * context on this CPU has cgroup events.
 378		 *
 379		 * ctx->nr_cgroups reports the number of cgroup
 380		 * events for a context.
 381		 */
 382		if (cpuctx->ctx.nr_cgroups > 0) {
 383			perf_ctx_lock(cpuctx, cpuctx->task_ctx);
 384			perf_pmu_disable(cpuctx->ctx.pmu);
 385
 386			if (mode & PERF_CGROUP_SWOUT) {
 387				cpu_ctx_sched_out(cpuctx, EVENT_ALL);
 388				/*
 389				 * must not be done before ctxswout due
 390				 * to event_filter_match() in event_sched_out()
 391				 */
 392				cpuctx->cgrp = NULL;
 393			}
 394
 395			if (mode & PERF_CGROUP_SWIN) {
 396				WARN_ON_ONCE(cpuctx->cgrp);
 397				/* set cgrp before ctxsw in to
 398				 * allow event_filter_match() to not
 399				 * have to pass task around
 400				 */
 401				cpuctx->cgrp = perf_cgroup_from_task(task);
 402				cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
 403			}
 404			perf_pmu_enable(cpuctx->ctx.pmu);
 405			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
 
 406		}
 
 
 407	}
 408
 409	rcu_read_unlock();
 410
 411	local_irq_restore(flags);
 412}
 413
 414static inline void perf_cgroup_sched_out(struct task_struct *task,
 415					 struct task_struct *next)
 416{
 417	struct perf_cgroup *cgrp1;
 418	struct perf_cgroup *cgrp2 = NULL;
 419
 
 420	/*
 421	 * we come here when we know perf_cgroup_events > 0
 
 
 422	 */
 423	cgrp1 = perf_cgroup_from_task(task);
 424
 425	/*
 426	 * next is NULL when called from perf_event_enable_on_exec()
 427	 * that will systematically cause a cgroup_switch()
 428	 */
 429	if (next)
 430		cgrp2 = perf_cgroup_from_task(next);
 431
 432	/*
 433	 * only schedule out current cgroup events if we know
 434	 * that we are switching to a different cgroup. Otherwise,
 435	 * do no touch the cgroup events.
 436	 */
 437	if (cgrp1 != cgrp2)
 438		perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
 
 
 439}
 440
 441static inline void perf_cgroup_sched_in(struct task_struct *prev,
 442					struct task_struct *task)
 443{
 444	struct perf_cgroup *cgrp1;
 445	struct perf_cgroup *cgrp2 = NULL;
 446
 
 447	/*
 448	 * we come here when we know perf_cgroup_events > 0
 
 
 449	 */
 450	cgrp1 = perf_cgroup_from_task(task);
 451
 452	/* prev can never be NULL */
 453	cgrp2 = perf_cgroup_from_task(prev);
 454
 455	/*
 456	 * only need to schedule in cgroup events if we are changing
 457	 * cgroup during ctxsw. Cgroup events were not scheduled
 458	 * out of ctxsw out if that was not the case.
 459	 */
 460	if (cgrp1 != cgrp2)
 461		perf_cgroup_switch(task, PERF_CGROUP_SWIN);
 
 
 462}
 463
 464static inline int perf_cgroup_connect(int fd, struct perf_event *event,
 465				      struct perf_event_attr *attr,
 466				      struct perf_event *group_leader)
 467{
 468	struct perf_cgroup *cgrp;
 469	struct cgroup_subsys_state *css;
 470	struct file *file;
 471	int ret = 0, fput_needed;
 472
 473	file = fget_light(fd, &fput_needed);
 474	if (!file)
 475		return -EBADF;
 476
 477	css = cgroup_css_from_dir(file, perf_subsys_id);
 
 478	if (IS_ERR(css)) {
 479		ret = PTR_ERR(css);
 480		goto out;
 481	}
 482
 483	cgrp = container_of(css, struct perf_cgroup, css);
 484	event->cgrp = cgrp;
 485
 486	/* must be done before we fput() the file */
 487	if (!perf_tryget_cgroup(event)) {
 488		event->cgrp = NULL;
 489		ret = -ENOENT;
 490		goto out;
 491	}
 492
 493	/*
 494	 * all events in a group must monitor
 495	 * the same cgroup because a task belongs
 496	 * to only one perf cgroup at a time
 497	 */
 498	if (group_leader && group_leader->cgrp != cgrp) {
 499		perf_detach_cgroup(event);
 500		ret = -EINVAL;
 501	}
 502out:
 503	fput_light(file, fput_needed);
 504	return ret;
 505}
 506
 507static inline void
 508perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
 509{
 510	struct perf_cgroup_info *t;
 511	t = per_cpu_ptr(event->cgrp->info, event->cpu);
 512	event->shadow_ctx_time = now - t->timestamp;
 513}
 514
 
 
 
 
 515static inline void
 516perf_cgroup_defer_enabled(struct perf_event *event)
 
 517{
 
 
 
 
 
 
 518	/*
 519	 * when the current task's perf cgroup does not match
 520	 * the event's, we need to remember to call the
 521	 * perf_mark_enable() function the first time a task with
 522	 * a matching perf cgroup is scheduled in.
 523	 */
 524	if (is_cgroup_event(event) && !perf_cgroup_match(event))
 525		event->cgrp_defer_enabled = 1;
 526}
 527
 528static inline void
 529perf_cgroup_mark_enabled(struct perf_event *event,
 530			 struct perf_event_context *ctx)
 531{
 532	struct perf_event *sub;
 533	u64 tstamp = perf_event_time(event);
 
 
 
 
 
 
 534
 535	if (!event->cgrp_defer_enabled)
 
 
 536		return;
 537
 538	event->cgrp_defer_enabled = 0;
 
 
 539
 540	event->tstamp_enabled = tstamp - event->total_time_enabled;
 541	list_for_each_entry(sub, &event->sibling_list, group_entry) {
 542		if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
 543			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
 544			sub->cgrp_defer_enabled = 0;
 545		}
 546	}
 547}
 
 548#else /* !CONFIG_CGROUP_PERF */
 549
 550static inline bool
 551perf_cgroup_match(struct perf_event *event)
 552{
 553	return true;
 554}
 555
 556static inline void perf_detach_cgroup(struct perf_event *event)
 557{}
 558
 559static inline int is_cgroup_event(struct perf_event *event)
 560{
 561	return 0;
 562}
 563
 564static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
 565{
 566	return 0;
 567}
 568
 569static inline void update_cgrp_time_from_event(struct perf_event *event)
 570{
 571}
 572
 573static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
 574{
 575}
 576
 577static inline void perf_cgroup_sched_out(struct task_struct *task,
 578					 struct task_struct *next)
 579{
 580}
 581
 582static inline void perf_cgroup_sched_in(struct task_struct *prev,
 583					struct task_struct *task)
 584{
 585}
 586
 587static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
 588				      struct perf_event_attr *attr,
 589				      struct perf_event *group_leader)
 590{
 591	return -EINVAL;
 592}
 593
 594static inline void
 595perf_cgroup_set_timestamp(struct task_struct *task,
 596			  struct perf_event_context *ctx)
 597{
 598}
 599
 600void
 601perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
 602{
 603}
 604
 605static inline void
 606perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
 607{
 608}
 609
 610static inline u64 perf_cgroup_event_time(struct perf_event *event)
 611{
 612	return 0;
 613}
 614
 615static inline void
 616perf_cgroup_defer_enabled(struct perf_event *event)
 
 617{
 618}
 619
 620static inline void
 621perf_cgroup_mark_enabled(struct perf_event *event,
 622			 struct perf_event_context *ctx)
 
 
 
 
 
 
 
 
 623{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 624}
 625#endif
 626
 627void perf_pmu_disable(struct pmu *pmu)
 628{
 629	int *count = this_cpu_ptr(pmu->pmu_disable_count);
 630	if (!(*count)++)
 631		pmu->pmu_disable(pmu);
 632}
 633
 634void perf_pmu_enable(struct pmu *pmu)
 635{
 636	int *count = this_cpu_ptr(pmu->pmu_disable_count);
 637	if (!--(*count))
 638		pmu->pmu_enable(pmu);
 639}
 640
 641static DEFINE_PER_CPU(struct list_head, rotation_list);
 642
 643/*
 644 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
 645 * because they're strictly cpu affine and rotate_start is called with IRQs
 646 * disabled, while rotate_context is called from IRQ context.
 
 647 */
 648static void perf_pmu_rotate_start(struct pmu *pmu)
 649{
 650	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
 651	struct list_head *head = &__get_cpu_var(rotation_list);
 652
 653	WARN_ON(!irqs_disabled());
 654
 655	if (list_empty(&cpuctx->rotation_list))
 656		list_add(&cpuctx->rotation_list, head);
 
 
 
 
 
 
 
 
 
 
 657}
 658
 659static void get_ctx(struct perf_event_context *ctx)
 660{
 661	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
 
 
 
 
 
 
 
 
 
 662}
 663
 664static void put_ctx(struct perf_event_context *ctx)
 665{
 666	if (atomic_dec_and_test(&ctx->refcount)) {
 667		if (ctx->parent_ctx)
 668			put_ctx(ctx->parent_ctx);
 669		if (ctx->task)
 670			put_task_struct(ctx->task);
 671		kfree_rcu(ctx, rcu_head);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 672	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 673}
 674
 675static void unclone_ctx(struct perf_event_context *ctx)
 
 
 
 
 
 
 676{
 677	if (ctx->parent_ctx) {
 678		put_ctx(ctx->parent_ctx);
 
 
 
 679		ctx->parent_ctx = NULL;
 680	}
 
 
 681}
 682
 683static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
 
 684{
 
 685	/*
 686	 * only top level events have the pid namespace they were created in
 687	 */
 688	if (event->parent)
 689		event = event->parent;
 690
 691	return task_tgid_nr_ns(p, event->ns);
 
 
 
 
 692}
 693
 694static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
 695{
 696	/*
 697	 * only top level events have the pid namespace they were created in
 698	 */
 699	if (event->parent)
 700		event = event->parent;
 701
 702	return task_pid_nr_ns(p, event->ns);
 
 
 703}
 704
 705/*
 706 * If we inherit events we want to return the parent event id
 707 * to userspace.
 708 */
 709static u64 primary_event_id(struct perf_event *event)
 710{
 711	u64 id = event->id;
 712
 713	if (event->parent)
 714		id = event->parent->id;
 715
 716	return id;
 717}
 718
 719/*
 720 * Get the perf_event_context for a task and lock it.
 
 721 * This has to cope with with the fact that until it is locked,
 722 * the context could get moved to another task.
 723 */
 724static struct perf_event_context *
 725perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
 726{
 727	struct perf_event_context *ctx;
 728
 729	rcu_read_lock();
 730retry:
 
 
 
 
 
 
 
 
 
 
 
 731	ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
 732	if (ctx) {
 733		/*
 734		 * If this context is a clone of another, it might
 735		 * get swapped for another underneath us by
 736		 * perf_event_task_sched_out, though the
 737		 * rcu_read_lock() protects us from any context
 738		 * getting freed.  Lock the context and check if it
 739		 * got swapped before we could get the lock, and retry
 740		 * if so.  If we locked the right context, then it
 741		 * can't get swapped on us any more.
 742		 */
 743		raw_spin_lock_irqsave(&ctx->lock, *flags);
 744		if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
 745			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
 
 
 746			goto retry;
 747		}
 748
 749		if (!atomic_inc_not_zero(&ctx->refcount)) {
 750			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
 
 751			ctx = NULL;
 
 
 752		}
 753	}
 754	rcu_read_unlock();
 
 
 755	return ctx;
 756}
 757
 758/*
 759 * Get the context for a task and increment its pin_count so it
 760 * can't get swapped to another task.  This also increments its
 761 * reference count so that the context can't get freed.
 762 */
 763static struct perf_event_context *
 764perf_pin_task_context(struct task_struct *task, int ctxn)
 765{
 766	struct perf_event_context *ctx;
 767	unsigned long flags;
 768
 769	ctx = perf_lock_task_context(task, ctxn, &flags);
 770	if (ctx) {
 771		++ctx->pin_count;
 772		raw_spin_unlock_irqrestore(&ctx->lock, flags);
 773	}
 774	return ctx;
 775}
 776
 777static void perf_unpin_context(struct perf_event_context *ctx)
 778{
 779	unsigned long flags;
 780
 781	raw_spin_lock_irqsave(&ctx->lock, flags);
 782	--ctx->pin_count;
 783	raw_spin_unlock_irqrestore(&ctx->lock, flags);
 784}
 785
 786/*
 787 * Update the record of the current time in a context.
 788 */
 789static void update_context_time(struct perf_event_context *ctx)
 790{
 791	u64 now = perf_clock();
 792
 793	ctx->time += now - ctx->timestamp;
 794	ctx->timestamp = now;
 795}
 796
 797static u64 perf_event_time(struct perf_event *event)
 798{
 799	struct perf_event_context *ctx = event->ctx;
 800
 801	if (is_cgroup_event(event))
 802		return perf_cgroup_event_time(event);
 803
 804	return ctx ? ctx->time : 0;
 805}
 806
 807/*
 808 * Update the total_time_enabled and total_time_running fields for a event.
 809 * The caller of this function needs to hold the ctx->lock.
 810 */
 811static void update_event_times(struct perf_event *event)
 812{
 813	struct perf_event_context *ctx = event->ctx;
 814	u64 run_end;
 
 
 815
 816	if (event->state < PERF_EVENT_STATE_INACTIVE ||
 817	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
 818		return;
 819	/*
 820	 * in cgroup mode, time_enabled represents
 821	 * the time the event was enabled AND active
 822	 * tasks were in the monitored cgroup. This is
 823	 * independent of the activity of the context as
 824	 * there may be a mix of cgroup and non-cgroup events.
 825	 *
 826	 * That is why we treat cgroup events differently
 827	 * here.
 828	 */
 829	if (is_cgroup_event(event))
 830		run_end = perf_cgroup_event_time(event);
 831	else if (ctx->is_active)
 832		run_end = ctx->time;
 833	else
 834		run_end = event->tstamp_stopped;
 835
 836	event->total_time_enabled = run_end - event->tstamp_enabled;
 837
 838	if (event->state == PERF_EVENT_STATE_INACTIVE)
 839		run_end = event->tstamp_stopped;
 840	else
 841		run_end = perf_event_time(event);
 842
 843	event->total_time_running = run_end - event->tstamp_running;
 
 
 844
 
 845}
 846
 847/*
 848 * Update total_time_enabled and total_time_running for all events in a group.
 849 */
 850static void update_group_times(struct perf_event *leader)
 851{
 852	struct perf_event *event;
 853
 854	update_event_times(leader);
 855	list_for_each_entry(event, &leader->sibling_list, group_entry)
 856		update_event_times(event);
 857}
 858
 859static struct list_head *
 860ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
 
 
 
 
 861{
 862	if (event->attr.pinned)
 863		return &ctx->pinned_groups;
 864	else
 865		return &ctx->flexible_groups;
 866}
 867
 868/*
 869 * Add a event from the lists for its context.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 870 * Must be called with ctx->mutex and ctx->lock held.
 871 */
 872static void
 873list_add_event(struct perf_event *event, struct perf_event_context *ctx)
 874{
 
 
 875	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
 876	event->attach_state |= PERF_ATTACH_CONTEXT;
 877
 
 
 878	/*
 879	 * If we're a stand alone event or group leader, we go to the context
 880	 * list, group events are kept attached to the group so that
 881	 * perf_group_detach can, at all times, locate all siblings.
 882	 */
 883	if (event->group_leader == event) {
 884		struct list_head *list;
 885
 886		if (is_software_event(event))
 887			event->group_flags |= PERF_GROUP_SOFTWARE;
 888
 889		list = ctx_group_list(event, ctx);
 890		list_add_tail(&event->group_entry, list);
 891	}
 892
 893	if (is_cgroup_event(event))
 894		ctx->nr_cgroups++;
 895
 896	if (has_branch_stack(event))
 897		ctx->nr_branch_stack++;
 898
 899	list_add_rcu(&event->event_entry, &ctx->event_list);
 900	if (!ctx->nr_events)
 901		perf_pmu_rotate_start(ctx->pmu);
 902	ctx->nr_events++;
 903	if (event->attr.inherit_stat)
 904		ctx->nr_stat++;
 
 
 905}
 906
 907/*
 908 * Called at perf_event creation and when events are attached/detached from a
 909 * group.
 910 */
 911static void perf_event__read_size(struct perf_event *event)
 
 
 
 
 
 
 912{
 913	int entry = sizeof(u64); /* value */
 914	int size = 0;
 915	int nr = 1;
 916
 917	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
 918		size += sizeof(u64);
 919
 920	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
 921		size += sizeof(u64);
 922
 923	if (event->attr.read_format & PERF_FORMAT_ID)
 924		entry += sizeof(u64);
 925
 926	if (event->attr.read_format & PERF_FORMAT_GROUP) {
 927		nr += event->group_leader->nr_siblings;
 928		size += sizeof(u64);
 929	}
 930
 931	size += entry * nr;
 932	event->read_size = size;
 933}
 934
 935static void perf_event__header_size(struct perf_event *event)
 936{
 937	struct perf_sample_data *data;
 938	u64 sample_type = event->attr.sample_type;
 939	u16 size = 0;
 940
 941	perf_event__read_size(event);
 942
 943	if (sample_type & PERF_SAMPLE_IP)
 944		size += sizeof(data->ip);
 945
 946	if (sample_type & PERF_SAMPLE_ADDR)
 947		size += sizeof(data->addr);
 948
 949	if (sample_type & PERF_SAMPLE_PERIOD)
 950		size += sizeof(data->period);
 951
 
 
 
 952	if (sample_type & PERF_SAMPLE_READ)
 953		size += event->read_size;
 954
 
 
 
 
 
 
 
 
 
 955	event->header_size = size;
 956}
 957
 
 
 
 
 
 
 
 
 
 
 
 958static void perf_event__id_header_size(struct perf_event *event)
 959{
 960	struct perf_sample_data *data;
 961	u64 sample_type = event->attr.sample_type;
 962	u16 size = 0;
 963
 964	if (sample_type & PERF_SAMPLE_TID)
 965		size += sizeof(data->tid_entry);
 966
 967	if (sample_type & PERF_SAMPLE_TIME)
 968		size += sizeof(data->time);
 969
 
 
 
 970	if (sample_type & PERF_SAMPLE_ID)
 971		size += sizeof(data->id);
 972
 973	if (sample_type & PERF_SAMPLE_STREAM_ID)
 974		size += sizeof(data->stream_id);
 975
 976	if (sample_type & PERF_SAMPLE_CPU)
 977		size += sizeof(data->cpu_entry);
 978
 979	event->id_header_size = size;
 980}
 981
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 982static void perf_group_attach(struct perf_event *event)
 983{
 984	struct perf_event *group_leader = event->group_leader, *pos;
 985
 
 
 986	/*
 987	 * We can have double attach due to group movement in perf_event_open.
 988	 */
 989	if (event->attach_state & PERF_ATTACH_GROUP)
 990		return;
 991
 992	event->attach_state |= PERF_ATTACH_GROUP;
 993
 994	if (group_leader == event)
 995		return;
 996
 997	if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
 998			!is_software_event(event))
 999		group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1000
1001	list_add_tail(&event->group_entry, &group_leader->sibling_list);
 
 
1002	group_leader->nr_siblings++;
1003
1004	perf_event__header_size(group_leader);
1005
1006	list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1007		perf_event__header_size(pos);
1008}
1009
1010/*
1011 * Remove a event from the lists for its context.
1012 * Must be called with ctx->mutex and ctx->lock held.
1013 */
1014static void
1015list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1016{
1017	struct perf_cpu_context *cpuctx;
 
 
1018	/*
1019	 * We can have double detach due to exit/hot-unplug + close.
1020	 */
1021	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1022		return;
1023
1024	event->attach_state &= ~PERF_ATTACH_CONTEXT;
1025
1026	if (is_cgroup_event(event)) {
1027		ctx->nr_cgroups--;
1028		cpuctx = __get_cpu_context(ctx);
1029		/*
1030		 * if there are no more cgroup events
1031		 * then cler cgrp to avoid stale pointer
1032		 * in update_cgrp_time_from_cpuctx()
1033		 */
1034		if (!ctx->nr_cgroups)
1035			cpuctx->cgrp = NULL;
1036	}
1037
1038	if (has_branch_stack(event))
1039		ctx->nr_branch_stack--;
1040
1041	ctx->nr_events--;
1042	if (event->attr.inherit_stat)
1043		ctx->nr_stat--;
1044
1045	list_del_rcu(&event->event_entry);
1046
1047	if (event->group_leader == event)
1048		list_del_init(&event->group_entry);
1049
1050	update_group_times(event);
1051
1052	/*
1053	 * If event was in error state, then keep it
1054	 * that way, otherwise bogus counts will be
1055	 * returned on read(). The only way to get out
1056	 * of error state is by explicit re-enabling
1057	 * of the event
1058	 */
1059	if (event->state > PERF_EVENT_STATE_OFF)
1060		event->state = PERF_EVENT_STATE_OFF;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1061}
1062
1063static void perf_group_detach(struct perf_event *event)
1064{
1065	struct perf_event *sibling, *tmp;
1066	struct list_head *list = NULL;
 
 
1067
1068	/*
1069	 * We can have double detach due to exit/hot-unplug + close.
1070	 */
1071	if (!(event->attach_state & PERF_ATTACH_GROUP))
1072		return;
1073
1074	event->attach_state &= ~PERF_ATTACH_GROUP;
1075
 
 
1076	/*
1077	 * If this is a sibling, remove it from its group.
1078	 */
1079	if (event->group_leader != event) {
1080		list_del_init(&event->group_entry);
1081		event->group_leader->nr_siblings--;
1082		goto out;
1083	}
1084
1085	if (!list_empty(&event->group_entry))
1086		list = &event->group_entry;
1087
1088	/*
1089	 * If this was a group event with sibling events then
1090	 * upgrade the siblings to singleton events by adding them
1091	 * to whatever list we are on.
1092	 */
1093	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1094		if (list)
1095			list_move_tail(&sibling->group_entry, list);
1096		sibling->group_leader = sibling;
 
1097
1098		/* Inherit group flags from the previous leader */
1099		sibling->group_flags = event->group_flags;
 
 
 
 
 
 
 
 
 
 
 
 
 
1100	}
1101
1102out:
1103	perf_event__header_size(event->group_leader);
1104
1105	list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1106		perf_event__header_size(tmp);
1107}
1108
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1109static inline int
1110event_filter_match(struct perf_event *event)
1111{
1112	return (event->cpu == -1 || event->cpu == smp_processor_id())
1113	    && perf_cgroup_match(event);
1114}
1115
1116static void
1117event_sched_out(struct perf_event *event,
1118		  struct perf_cpu_context *cpuctx,
1119		  struct perf_event_context *ctx)
1120{
1121	u64 tstamp = perf_event_time(event);
1122	u64 delta;
1123	/*
1124	 * An event which could not be activated because of
1125	 * filter mismatch still needs to have its timings
1126	 * maintained, otherwise bogus information is return
1127	 * via read() for time_enabled, time_running:
1128	 */
1129	if (event->state == PERF_EVENT_STATE_INACTIVE
1130	    && !event_filter_match(event)) {
1131		delta = tstamp - event->tstamp_stopped;
1132		event->tstamp_running += delta;
1133		event->tstamp_stopped = tstamp;
1134	}
1135
1136	if (event->state != PERF_EVENT_STATE_ACTIVE)
1137		return;
1138
1139	event->state = PERF_EVENT_STATE_INACTIVE;
1140	if (event->pending_disable) {
1141		event->pending_disable = 0;
1142		event->state = PERF_EVENT_STATE_OFF;
1143	}
1144	event->tstamp_stopped = tstamp;
 
 
 
1145	event->pmu->del(event, 0);
1146	event->oncpu = -1;
1147
 
 
 
 
 
 
1148	if (!is_software_event(event))
1149		cpuctx->active_oncpu--;
1150	ctx->nr_active--;
 
1151	if (event->attr.freq && event->attr.sample_freq)
1152		ctx->nr_freq--;
1153	if (event->attr.exclusive || !cpuctx->active_oncpu)
1154		cpuctx->exclusive = 0;
 
 
1155}
1156
1157static void
1158group_sched_out(struct perf_event *group_event,
1159		struct perf_cpu_context *cpuctx,
1160		struct perf_event_context *ctx)
1161{
1162	struct perf_event *event;
1163	int state = group_event->state;
 
 
 
 
1164
1165	event_sched_out(group_event, cpuctx, ctx);
1166
1167	/*
1168	 * Schedule out siblings (if any):
1169	 */
1170	list_for_each_entry(event, &group_event->sibling_list, group_entry)
1171		event_sched_out(event, cpuctx, ctx);
1172
1173	if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
 
 
1174		cpuctx->exclusive = 0;
1175}
1176
 
 
1177/*
1178 * Cross CPU call to remove a performance event
1179 *
1180 * We disable the event on the hardware level first. After that we
1181 * remove it from the context list.
1182 */
1183static int __perf_remove_from_context(void *info)
 
 
 
 
1184{
1185	struct perf_event *event = info;
1186	struct perf_event_context *ctx = event->ctx;
1187	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
 
 
 
1188
1189	raw_spin_lock(&ctx->lock);
1190	event_sched_out(event, cpuctx, ctx);
 
 
1191	list_del_event(event, ctx);
1192	if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
 
1193		ctx->is_active = 0;
1194		cpuctx->task_ctx = NULL;
 
 
 
1195	}
1196	raw_spin_unlock(&ctx->lock);
1197
1198	return 0;
1199}
1200
1201
1202/*
1203 * Remove the event from a task's (or a CPU's) list of events.
1204 *
1205 * CPU events are removed with a smp call. For task events we only
1206 * call when the task is on a CPU.
1207 *
1208 * If event->ctx is a cloned context, callers must make sure that
1209 * every task struct that event->ctx->task could possibly point to
1210 * remains valid.  This is OK when called from perf_release since
1211 * that only calls us on the top-level context, which can't be a clone.
1212 * When called from perf_event_exit_task, it's OK because the
1213 * context has been detached from its task.
1214 */
1215static void perf_remove_from_context(struct perf_event *event)
1216{
1217	struct perf_event_context *ctx = event->ctx;
1218	struct task_struct *task = ctx->task;
1219
1220	lockdep_assert_held(&ctx->mutex);
1221
1222	if (!task) {
1223		/*
1224		 * Per cpu events are removed via an smp call and
1225		 * the removal is always successful.
1226		 */
1227		cpu_function_call(event->cpu, __perf_remove_from_context, event);
1228		return;
1229	}
1230
1231retry:
1232	if (!task_function_call(task, __perf_remove_from_context, event))
1233		return;
1234
1235	raw_spin_lock_irq(&ctx->lock);
1236	/*
1237	 * If we failed to find a running task, but find the context active now
1238	 * that we've acquired the ctx->lock, retry.
 
 
1239	 */
1240	if (ctx->is_active) {
 
 
 
 
 
 
 
 
1241		raw_spin_unlock_irq(&ctx->lock);
1242		goto retry;
1243	}
1244
1245	/*
1246	 * Since the task isn't running, its safe to remove the event, us
1247	 * holding the ctx->lock ensures the task won't get scheduled in.
1248	 */
1249	list_del_event(event, ctx);
1250	raw_spin_unlock_irq(&ctx->lock);
1251}
1252
1253/*
1254 * Cross CPU call to disable a performance event
1255 */
1256static int __perf_event_disable(void *info)
 
 
 
1257{
1258	struct perf_event *event = info;
1259	struct perf_event_context *ctx = event->ctx;
1260	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1261
1262	/*
1263	 * If this is a per-task event, need to check whether this
1264	 * event's task is the current task on this cpu.
1265	 *
1266	 * Can trigger due to concurrent perf_event_context_sched_out()
1267	 * flipping contexts around.
1268	 */
1269	if (ctx->task && cpuctx->task_ctx != ctx)
1270		return -EINVAL;
1271
1272	raw_spin_lock(&ctx->lock);
1273
1274	/*
1275	 * If the event is on, turn it off.
1276	 * If it is in error state, leave it in error state.
1277	 */
1278	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1279		update_context_time(ctx);
1280		update_cgrp_time_from_event(event);
1281		update_group_times(event);
1282		if (event == event->group_leader)
1283			group_sched_out(event, cpuctx, ctx);
1284		else
1285			event_sched_out(event, cpuctx, ctx);
1286		event->state = PERF_EVENT_STATE_OFF;
1287	}
1288
1289	raw_spin_unlock(&ctx->lock);
 
 
 
1290
1291	return 0;
1292}
1293
1294/*
1295 * Disable a event.
1296 *
1297 * If event->ctx is a cloned context, callers must make sure that
1298 * every task struct that event->ctx->task could possibly point to
1299 * remains valid.  This condition is satisifed when called through
1300 * perf_event_for_each_child or perf_event_for_each because they
1301 * hold the top-level event's child_mutex, so any descendant that
1302 * goes to exit will block in sync_child_event.
 
1303 * When called from perf_pending_event it's OK because event->ctx
1304 * is the current context on this CPU and preemption is disabled,
1305 * hence we can't get into perf_event_task_sched_out for this context.
1306 */
1307void perf_event_disable(struct perf_event *event)
1308{
1309	struct perf_event_context *ctx = event->ctx;
1310	struct task_struct *task = ctx->task;
1311
1312	if (!task) {
1313		/*
1314		 * Disable the event on the cpu that it's on
1315		 */
1316		cpu_function_call(event->cpu, __perf_event_disable, event);
1317		return;
1318	}
 
1319
1320retry:
1321	if (!task_function_call(task, __perf_event_disable, event))
1322		return;
1323
1324	raw_spin_lock_irq(&ctx->lock);
1325	/*
1326	 * If the event is still active, we need to retry the cross-call.
1327	 */
1328	if (event->state == PERF_EVENT_STATE_ACTIVE) {
1329		raw_spin_unlock_irq(&ctx->lock);
1330		/*
1331		 * Reload the task pointer, it might have been changed by
1332		 * a concurrent perf_event_context_sched_out().
1333		 */
1334		task = ctx->task;
1335		goto retry;
1336	}
1337
1338	/*
1339	 * Since we have the lock this context can't be scheduled
1340	 * in, so we can change the state safely.
1341	 */
1342	if (event->state == PERF_EVENT_STATE_INACTIVE) {
1343		update_group_times(event);
1344		event->state = PERF_EVENT_STATE_OFF;
1345	}
1346	raw_spin_unlock_irq(&ctx->lock);
 
 
1347}
1348EXPORT_SYMBOL_GPL(perf_event_disable);
1349
 
 
 
 
 
 
 
1350static void perf_set_shadow_time(struct perf_event *event,
1351				 struct perf_event_context *ctx,
1352				 u64 tstamp)
1353{
1354	/*
1355	 * use the correct time source for the time snapshot
1356	 *
1357	 * We could get by without this by leveraging the
1358	 * fact that to get to this function, the caller
1359	 * has most likely already called update_context_time()
1360	 * and update_cgrp_time_xx() and thus both timestamp
1361	 * are identical (or very close). Given that tstamp is,
1362	 * already adjusted for cgroup, we could say that:
1363	 *    tstamp - ctx->timestamp
1364	 * is equivalent to
1365	 *    tstamp - cgrp->timestamp.
1366	 *
1367	 * Then, in perf_output_read(), the calculation would
1368	 * work with no changes because:
1369	 * - event is guaranteed scheduled in
1370	 * - no scheduled out in between
1371	 * - thus the timestamp would be the same
1372	 *
1373	 * But this is a bit hairy.
1374	 *
1375	 * So instead, we have an explicit cgroup call to remain
1376	 * within the time time source all along. We believe it
1377	 * is cleaner and simpler to understand.
1378	 */
1379	if (is_cgroup_event(event))
1380		perf_cgroup_set_shadow_time(event, tstamp);
1381	else
1382		event->shadow_ctx_time = tstamp - ctx->timestamp;
1383}
1384
1385#define MAX_INTERRUPTS (~0ULL)
1386
1387static void perf_log_throttle(struct perf_event *event, int enable);
 
1388
1389static int
1390event_sched_in(struct perf_event *event,
1391		 struct perf_cpu_context *cpuctx,
1392		 struct perf_event_context *ctx)
1393{
1394	u64 tstamp = perf_event_time(event);
 
 
1395
1396	if (event->state <= PERF_EVENT_STATE_OFF)
1397		return 0;
1398
1399	event->state = PERF_EVENT_STATE_ACTIVE;
1400	event->oncpu = smp_processor_id();
 
 
 
 
 
 
1401
1402	/*
1403	 * Unthrottle events, since we scheduled we might have missed several
1404	 * ticks already, also for a heavily scheduling task there is little
1405	 * guarantee it'll get a tick in a timely manner.
1406	 */
1407	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1408		perf_log_throttle(event, 1);
1409		event->hw.interrupts = 0;
1410	}
1411
1412	/*
1413	 * The new state must be visible before we turn it on in the hardware:
1414	 */
1415	smp_wmb();
 
1416
1417	if (event->pmu->add(event, PERF_EF_START)) {
1418		event->state = PERF_EVENT_STATE_INACTIVE;
1419		event->oncpu = -1;
1420		return -EAGAIN;
 
1421	}
1422
1423	event->tstamp_running += tstamp - event->tstamp_stopped;
1424
1425	perf_set_shadow_time(event, ctx, tstamp);
1426
1427	if (!is_software_event(event))
1428		cpuctx->active_oncpu++;
1429	ctx->nr_active++;
 
1430	if (event->attr.freq && event->attr.sample_freq)
1431		ctx->nr_freq++;
1432
1433	if (event->attr.exclusive)
1434		cpuctx->exclusive = 1;
1435
1436	return 0;
 
 
 
1437}
1438
1439static int
1440group_sched_in(struct perf_event *group_event,
1441	       struct perf_cpu_context *cpuctx,
1442	       struct perf_event_context *ctx)
1443{
1444	struct perf_event *event, *partial_group = NULL;
1445	struct pmu *pmu = group_event->pmu;
1446	u64 now = ctx->time;
1447	bool simulate = false;
1448
1449	if (group_event->state == PERF_EVENT_STATE_OFF)
1450		return 0;
1451
1452	pmu->start_txn(pmu);
1453
1454	if (event_sched_in(group_event, cpuctx, ctx)) {
1455		pmu->cancel_txn(pmu);
 
1456		return -EAGAIN;
1457	}
1458
1459	/*
1460	 * Schedule in siblings as one group (if any):
1461	 */
1462	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1463		if (event_sched_in(event, cpuctx, ctx)) {
1464			partial_group = event;
1465			goto group_error;
1466		}
1467	}
1468
1469	if (!pmu->commit_txn(pmu))
1470		return 0;
1471
1472group_error:
1473	/*
1474	 * Groups can be scheduled in as one unit only, so undo any
1475	 * partial group before returning:
1476	 * The events up to the failed event are scheduled out normally,
1477	 * tstamp_stopped will be updated.
1478	 *
1479	 * The failed events and the remaining siblings need to have
1480	 * their timings updated as if they had gone thru event_sched_in()
1481	 * and event_sched_out(). This is required to get consistent timings
1482	 * across the group. This also takes care of the case where the group
1483	 * could never be scheduled by ensuring tstamp_stopped is set to mark
1484	 * the time the event was actually stopped, such that time delta
1485	 * calculation in update_event_times() is correct.
1486	 */
1487	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1488		if (event == partial_group)
1489			simulate = true;
1490
1491		if (simulate) {
1492			event->tstamp_running += now - event->tstamp_stopped;
1493			event->tstamp_stopped = now;
1494		} else {
1495			event_sched_out(event, cpuctx, ctx);
1496		}
1497	}
1498	event_sched_out(group_event, cpuctx, ctx);
1499
1500	pmu->cancel_txn(pmu);
1501
 
 
1502	return -EAGAIN;
1503}
1504
1505/*
1506 * Work out whether we can put this event group on the CPU now.
1507 */
1508static int group_can_go_on(struct perf_event *event,
1509			   struct perf_cpu_context *cpuctx,
1510			   int can_add_hw)
1511{
1512	/*
1513	 * Groups consisting entirely of software events can always go on.
1514	 */
1515	if (event->group_flags & PERF_GROUP_SOFTWARE)
1516		return 1;
1517	/*
1518	 * If an exclusive group is already on, no other hardware
1519	 * events can go on.
1520	 */
1521	if (cpuctx->exclusive)
1522		return 0;
1523	/*
1524	 * If this group is exclusive and there are already
1525	 * events on the CPU, it can't go on.
1526	 */
1527	if (event->attr.exclusive && cpuctx->active_oncpu)
1528		return 0;
1529	/*
1530	 * Otherwise, try to add it if all previous groups were able
1531	 * to go on.
1532	 */
1533	return can_add_hw;
1534}
1535
1536static void add_event_to_ctx(struct perf_event *event,
1537			       struct perf_event_context *ctx)
1538{
1539	u64 tstamp = perf_event_time(event);
1540
1541	list_add_event(event, ctx);
1542	perf_group_attach(event);
1543	event->tstamp_enabled = tstamp;
1544	event->tstamp_running = tstamp;
1545	event->tstamp_stopped = tstamp;
1546}
1547
1548static void task_ctx_sched_out(struct perf_event_context *ctx);
 
 
1549static void
1550ctx_sched_in(struct perf_event_context *ctx,
1551	     struct perf_cpu_context *cpuctx,
1552	     enum event_type_t event_type,
1553	     struct task_struct *task);
1554
 
 
 
 
 
 
 
 
 
 
 
 
 
1555static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1556				struct perf_event_context *ctx,
1557				struct task_struct *task)
1558{
1559	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1560	if (ctx)
1561		ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1562	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1563	if (ctx)
1564		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1565}
1566
1567/*
1568 * Cross CPU call to install and enable a performance event
 
 
 
 
1569 *
1570 * Must be called with ctx->mutex held
 
 
 
 
 
 
1571 */
1572static int  __perf_install_in_context(void *info)
 
 
1573{
1574	struct perf_event *event = info;
1575	struct perf_event_context *ctx = event->ctx;
1576	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1577	struct perf_event_context *task_ctx = cpuctx->task_ctx;
1578	struct task_struct *task = current;
1579
1580	perf_ctx_lock(cpuctx, task_ctx);
1581	perf_pmu_disable(cpuctx->ctx.pmu);
1582
1583	/*
1584	 * If there was an active task_ctx schedule it out.
 
1585	 */
 
 
 
 
 
 
1586	if (task_ctx)
1587		task_ctx_sched_out(task_ctx);
1588
1589	/*
1590	 * If the context we're installing events in is not the
1591	 * active task_ctx, flip them.
 
 
 
1592	 */
1593	if (ctx->task && task_ctx != ctx) {
1594		if (task_ctx)
1595			raw_spin_unlock(&task_ctx->lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1596		raw_spin_lock(&ctx->lock);
1597		task_ctx = ctx;
1598	}
1599
1600	if (task_ctx) {
1601		cpuctx->task_ctx = task_ctx;
1602		task = task_ctx->task;
1603	}
1604
1605	cpu_ctx_sched_out(cpuctx, EVENT_ALL);
 
 
 
 
 
 
 
 
 
 
1606
1607	update_context_time(ctx);
1608	/*
1609	 * update cgrp time only if current cgrp
1610	 * matches event->cgrp. Must be done before
1611	 * calling add_event_to_ctx()
1612	 */
1613	update_cgrp_time_from_event(event);
1614
1615	add_event_to_ctx(event, ctx);
 
 
 
 
 
 
 
 
 
 
1616
1617	/*
1618	 * Schedule everything back in
1619	 */
1620	perf_event_sched_in(cpuctx, task_ctx, task);
 
 
 
1621
1622	perf_pmu_enable(cpuctx->ctx.pmu);
1623	perf_ctx_unlock(cpuctx, task_ctx);
1624
1625	return 0;
1626}
1627
 
 
 
1628/*
1629 * Attach a performance event to a context
1630 *
1631 * First we add the event to the list with the hardware enable bit
1632 * in event->hw_config cleared.
1633 *
1634 * If the event is attached to a task which is on a CPU we use a smp
1635 * call to enable it in the task context. The task might have been
1636 * scheduled away, but we check this in the smp call again.
1637 */
1638static void
1639perf_install_in_context(struct perf_event_context *ctx,
1640			struct perf_event *event,
1641			int cpu)
1642{
1643	struct task_struct *task = ctx->task;
1644
1645	lockdep_assert_held(&ctx->mutex);
1646
1647	event->ctx = ctx;
 
 
 
 
 
 
 
 
 
1648
1649	if (!task) {
1650		/*
1651		 * Per cpu events are installed via an smp call and
1652		 * the install is always successful.
1653		 */
1654		cpu_function_call(cpu, __perf_install_in_context, event);
1655		return;
1656	}
1657
1658retry:
1659	if (!task_function_call(task, __perf_install_in_context, event))
 
 
1660		return;
1661
1662	raw_spin_lock_irq(&ctx->lock);
1663	/*
1664	 * If we failed to find a running task, but find the context active now
1665	 * that we've acquired the ctx->lock, retry.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1666	 */
1667	if (ctx->is_active) {
 
 
 
 
 
 
 
 
 
 
 
 
1668		raw_spin_unlock_irq(&ctx->lock);
1669		goto retry;
1670	}
1671
1672	/*
1673	 * Since the task isn't running, its safe to add the event, us holding
1674	 * the ctx->lock ensures the task won't get scheduled in.
1675	 */
 
 
 
 
1676	add_event_to_ctx(event, ctx);
1677	raw_spin_unlock_irq(&ctx->lock);
1678}
1679
1680/*
1681 * Put a event into inactive state and update time fields.
1682 * Enabling the leader of a group effectively enables all
1683 * the group members that aren't explicitly disabled, so we
1684 * have to update their ->tstamp_enabled also.
1685 * Note: this works for group members as well as group leaders
1686 * since the non-leader members' sibling_lists will be empty.
1687 */
1688static void __perf_event_mark_enabled(struct perf_event *event)
1689{
1690	struct perf_event *sub;
1691	u64 tstamp = perf_event_time(event);
1692
1693	event->state = PERF_EVENT_STATE_INACTIVE;
1694	event->tstamp_enabled = tstamp - event->total_time_enabled;
1695	list_for_each_entry(sub, &event->sibling_list, group_entry) {
1696		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1697			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
1698	}
1699}
1700
1701/*
1702 * Cross CPU call to enable a performance event
1703 */
1704static int __perf_event_enable(void *info)
 
 
 
1705{
1706	struct perf_event *event = info;
1707	struct perf_event_context *ctx = event->ctx;
1708	struct perf_event *leader = event->group_leader;
1709	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1710	int err;
1711
1712	if (WARN_ON_ONCE(!ctx->is_active))
1713		return -EINVAL;
 
1714
1715	raw_spin_lock(&ctx->lock);
1716	update_context_time(ctx);
1717
1718	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1719		goto unlock;
1720
1721	/*
1722	 * set current task's cgroup time reference point
1723	 */
1724	perf_cgroup_set_timestamp(current, ctx);
1725
1726	__perf_event_mark_enabled(event);
1727
1728	if (!event_filter_match(event)) {
1729		if (is_cgroup_event(event))
1730			perf_cgroup_defer_enabled(event);
1731		goto unlock;
1732	}
1733
1734	/*
1735	 * If the event is in a group and isn't the group leader,
1736	 * then don't put it on unless the group is on.
1737	 */
1738	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1739		goto unlock;
1740
1741	if (!group_can_go_on(event, cpuctx, 1)) {
1742		err = -EEXIST;
1743	} else {
1744		if (event == leader)
1745			err = group_sched_in(event, cpuctx, ctx);
1746		else
1747			err = event_sched_in(event, cpuctx, ctx);
1748	}
1749
1750	if (err) {
1751		/*
1752		 * If this event can't go on and it's part of a
1753		 * group, then the whole group has to come off.
1754		 */
1755		if (leader != event)
1756			group_sched_out(leader, cpuctx, ctx);
1757		if (leader->attr.pinned) {
1758			update_group_times(leader);
1759			leader->state = PERF_EVENT_STATE_ERROR;
1760		}
1761	}
1762
1763unlock:
1764	raw_spin_unlock(&ctx->lock);
 
1765
1766	return 0;
1767}
1768
1769/*
1770 * Enable a event.
1771 *
1772 * If event->ctx is a cloned context, callers must make sure that
1773 * every task struct that event->ctx->task could possibly point to
1774 * remains valid.  This condition is satisfied when called through
1775 * perf_event_for_each_child or perf_event_for_each as described
1776 * for perf_event_disable.
1777 */
1778void perf_event_enable(struct perf_event *event)
1779{
1780	struct perf_event_context *ctx = event->ctx;
1781	struct task_struct *task = ctx->task;
1782
1783	if (!task) {
1784		/*
1785		 * Enable the event on the cpu that it's on
1786		 */
1787		cpu_function_call(event->cpu, __perf_event_enable, event);
1788		return;
1789	}
1790
1791	raw_spin_lock_irq(&ctx->lock);
1792	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1793		goto out;
1794
1795	/*
1796	 * If the event is in error state, clear that first.
1797	 * That way, if we see the event in error state below, we
1798	 * know that it has gone back into error state, as distinct
1799	 * from the task having been scheduled away before the
1800	 * cross-call arrived.
1801	 */
1802	if (event->state == PERF_EVENT_STATE_ERROR)
1803		event->state = PERF_EVENT_STATE_OFF;
 
1804
1805retry:
1806	if (!ctx->is_active) {
1807		__perf_event_mark_enabled(event);
1808		goto out;
1809	}
1810
1811	raw_spin_unlock_irq(&ctx->lock);
 
 
 
 
 
1812
1813	if (!task_function_call(task, __perf_event_enable, event))
1814		return;
 
 
 
1815
1816	raw_spin_lock_irq(&ctx->lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1817
1818	/*
1819	 * If the context is active and the event is still off,
1820	 * we need to retry the cross-call.
1821	 */
1822	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1823		/*
1824		 * task could have been flipped by a concurrent
1825		 * perf_event_context_sched_out()
 
1826		 */
1827		task = ctx->task;
1828		goto retry;
1829	}
1830
1831out:
1832	raw_spin_unlock_irq(&ctx->lock);
1833}
1834EXPORT_SYMBOL_GPL(perf_event_enable);
1835
1836int perf_event_refresh(struct perf_event *event, int refresh)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1837{
1838	/*
1839	 * not supported on inherited events
1840	 */
1841	if (event->attr.inherit || !is_sampling_event(event))
1842		return -EINVAL;
1843
1844	atomic_add(refresh, &event->event_limit);
1845	perf_event_enable(event);
1846
1847	return 0;
1848}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1849EXPORT_SYMBOL_GPL(perf_event_refresh);
1850
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1851static void ctx_sched_out(struct perf_event_context *ctx,
1852			  struct perf_cpu_context *cpuctx,
1853			  enum event_type_t event_type)
1854{
1855	struct perf_event *event;
1856	int is_active = ctx->is_active;
1857
1858	ctx->is_active &= ~event_type;
1859	if (likely(!ctx->nr_events))
 
 
 
 
 
 
 
1860		return;
 
1861
1862	update_context_time(ctx);
1863	update_cgrp_time_from_cpuctx(cpuctx);
1864	if (!ctx->nr_active)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1865		return;
1866
 
 
 
 
 
 
1867	perf_pmu_disable(ctx->pmu);
1868	if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
1869		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1870			group_sched_out(event, cpuctx, ctx);
1871	}
1872
1873	if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
1874		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1875			group_sched_out(event, cpuctx, ctx);
1876	}
1877	perf_pmu_enable(ctx->pmu);
1878}
1879
1880/*
1881 * Test whether two contexts are equivalent, i.e. whether they
1882 * have both been cloned from the same version of the same context
1883 * and they both have the same number of enabled events.
1884 * If the number of enabled events is the same, then the set
1885 * of enabled events should be the same, because these are both
1886 * inherited contexts, therefore we can't access individual events
1887 * in them directly with an fd; we can only enable/disable all
1888 * events via prctl, or enable/disable all events in a family
1889 * via ioctl, which will have the same effect on both contexts.
1890 */
1891static int context_equiv(struct perf_event_context *ctx1,
1892			 struct perf_event_context *ctx2)
1893{
1894	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1895		&& ctx1->parent_gen == ctx2->parent_gen
1896		&& !ctx1->pin_count && !ctx2->pin_count;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1897}
1898
1899static void __perf_event_sync_stat(struct perf_event *event,
1900				     struct perf_event *next_event)
1901{
1902	u64 value;
1903
1904	if (!event->attr.inherit_stat)
1905		return;
1906
1907	/*
1908	 * Update the event value, we cannot use perf_event_read()
1909	 * because we're in the middle of a context switch and have IRQs
1910	 * disabled, which upsets smp_call_function_single(), however
1911	 * we know the event must be on the current CPU, therefore we
1912	 * don't need to use it.
1913	 */
1914	switch (event->state) {
1915	case PERF_EVENT_STATE_ACTIVE:
1916		event->pmu->read(event);
1917		/* fall-through */
1918
1919	case PERF_EVENT_STATE_INACTIVE:
1920		update_event_times(event);
1921		break;
1922
1923	default:
1924		break;
1925	}
1926
1927	/*
1928	 * In order to keep per-task stats reliable we need to flip the event
1929	 * values when we flip the contexts.
1930	 */
1931	value = local64_read(&next_event->count);
1932	value = local64_xchg(&event->count, value);
1933	local64_set(&next_event->count, value);
1934
1935	swap(event->total_time_enabled, next_event->total_time_enabled);
1936	swap(event->total_time_running, next_event->total_time_running);
1937
1938	/*
1939	 * Since we swizzled the values, update the user visible data too.
1940	 */
1941	perf_event_update_userpage(event);
1942	perf_event_update_userpage(next_event);
1943}
1944
1945#define list_next_entry(pos, member) \
1946	list_entry(pos->member.next, typeof(*pos), member)
1947
1948static void perf_event_sync_stat(struct perf_event_context *ctx,
1949				   struct perf_event_context *next_ctx)
1950{
1951	struct perf_event *event, *next_event;
1952
1953	if (!ctx->nr_stat)
1954		return;
1955
1956	update_context_time(ctx);
1957
1958	event = list_first_entry(&ctx->event_list,
1959				   struct perf_event, event_entry);
1960
1961	next_event = list_first_entry(&next_ctx->event_list,
1962					struct perf_event, event_entry);
1963
1964	while (&event->event_entry != &ctx->event_list &&
1965	       &next_event->event_entry != &next_ctx->event_list) {
1966
1967		__perf_event_sync_stat(event, next_event);
1968
1969		event = list_next_entry(event, event_entry);
1970		next_event = list_next_entry(next_event, event_entry);
1971	}
1972}
1973
1974static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1975					 struct task_struct *next)
1976{
1977	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1978	struct perf_event_context *next_ctx;
1979	struct perf_event_context *parent;
1980	struct perf_cpu_context *cpuctx;
1981	int do_switch = 1;
1982
1983	if (likely(!ctx))
1984		return;
1985
1986	cpuctx = __get_cpu_context(ctx);
1987	if (!cpuctx->task_ctx)
1988		return;
1989
1990	rcu_read_lock();
1991	parent = rcu_dereference(ctx->parent_ctx);
1992	next_ctx = next->perf_event_ctxp[ctxn];
1993	if (parent && next_ctx &&
1994	    rcu_dereference(next_ctx->parent_ctx) == parent) {
 
 
 
 
 
 
 
 
 
1995		/*
1996		 * Looks like the two contexts are clones, so we might be
1997		 * able to optimize the context switch.  We lock both
1998		 * contexts and check that they are clones under the
1999		 * lock (including re-checking that neither has been
2000		 * uncloned in the meantime).  It doesn't matter which
2001		 * order we take the locks because no other cpu could
2002		 * be trying to lock both of these tasks.
2003		 */
2004		raw_spin_lock(&ctx->lock);
2005		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2006		if (context_equiv(ctx, next_ctx)) {
 
 
 
 
 
2007			/*
2008			 * XXX do we need a memory barrier of sorts
2009			 * wrt to rcu_dereference() of perf_event_ctxp
 
 
 
2010			 */
2011			task->perf_event_ctxp[ctxn] = next_ctx;
2012			next->perf_event_ctxp[ctxn] = ctx;
2013			ctx->task = next;
2014			next_ctx->task = task;
2015			do_switch = 0;
2016
2017			perf_event_sync_stat(ctx, next_ctx);
2018		}
2019		raw_spin_unlock(&next_ctx->lock);
2020		raw_spin_unlock(&ctx->lock);
2021	}
 
2022	rcu_read_unlock();
2023
2024	if (do_switch) {
2025		raw_spin_lock(&ctx->lock);
2026		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2027		cpuctx->task_ctx = NULL;
2028		raw_spin_unlock(&ctx->lock);
2029	}
2030}
2031
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2032#define for_each_task_context_nr(ctxn)					\
2033	for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2034
2035/*
2036 * Called from scheduler to remove the events of the current task,
2037 * with interrupts disabled.
2038 *
2039 * We stop each event and update the event value in event->count.
2040 *
2041 * This does not protect us against NMI, but disable()
2042 * sets the disabled bit in the control field of event _before_
2043 * accessing the event control register. If a NMI hits, then it will
2044 * not restart the event.
2045 */
2046void __perf_event_task_sched_out(struct task_struct *task,
2047				 struct task_struct *next)
2048{
2049	int ctxn;
2050
 
 
 
 
 
 
2051	for_each_task_context_nr(ctxn)
2052		perf_event_context_sched_out(task, ctxn, next);
2053
2054	/*
2055	 * if cgroup events exist on this CPU, then we need
2056	 * to check if we have to switch out PMU state.
2057	 * cgroup event are system-wide mode only
2058	 */
2059	if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2060		perf_cgroup_sched_out(task, next);
2061}
2062
2063static void task_ctx_sched_out(struct perf_event_context *ctx)
2064{
2065	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2066
2067	if (!cpuctx->task_ctx)
2068		return;
2069
2070	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2071		return;
2072
2073	ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2074	cpuctx->task_ctx = NULL;
2075}
2076
2077/*
2078 * Called with IRQs disabled
2079 */
2080static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2081			      enum event_type_t event_type)
2082{
2083	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2084}
2085
2086static void
2087ctx_pinned_sched_in(struct perf_event_context *ctx,
2088		    struct perf_cpu_context *cpuctx)
2089{
2090	struct perf_event *event;
 
2091
2092	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2093		if (event->state <= PERF_EVENT_STATE_OFF)
2094			continue;
2095		if (!event_filter_match(event))
2096			continue;
2097
2098		/* may need to reset tstamp_enabled */
2099		if (is_cgroup_event(event))
2100			perf_cgroup_mark_enabled(event, ctx);
 
 
 
 
 
 
 
 
2101
2102		if (group_can_go_on(event, cpuctx, 1))
2103			group_sched_in(event, cpuctx, ctx);
 
2104
2105		/*
2106		 * If this pinned group hasn't been scheduled,
2107		 * put it in error state.
2108		 */
2109		if (event->state == PERF_EVENT_STATE_INACTIVE) {
2110			update_group_times(event);
2111			event->state = PERF_EVENT_STATE_ERROR;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2112		}
 
2113	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2114}
2115
2116static void
2117ctx_flexible_sched_in(struct perf_event_context *ctx,
2118		      struct perf_cpu_context *cpuctx)
2119{
2120	struct perf_event *event;
2121	int can_add_hw = 1;
2122
2123	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2124		/* Ignore events in OFF or ERROR state */
2125		if (event->state <= PERF_EVENT_STATE_OFF)
2126			continue;
2127		/*
2128		 * Listen to the 'cpu' scheduling filter constraint
2129		 * of events:
2130		 */
2131		if (!event_filter_match(event))
2132			continue;
2133
2134		/* may need to reset tstamp_enabled */
2135		if (is_cgroup_event(event))
2136			perf_cgroup_mark_enabled(event, ctx);
2137
2138		if (group_can_go_on(event, cpuctx, can_add_hw)) {
2139			if (group_sched_in(event, cpuctx, ctx))
2140				can_add_hw = 0;
2141		}
2142	}
2143}
2144
2145static void
2146ctx_sched_in(struct perf_event_context *ctx,
2147	     struct perf_cpu_context *cpuctx,
2148	     enum event_type_t event_type,
2149	     struct task_struct *task)
2150{
2151	u64 now;
2152	int is_active = ctx->is_active;
 
 
 
2153
2154	ctx->is_active |= event_type;
2155	if (likely(!ctx->nr_events))
2156		return;
2157
2158	now = perf_clock();
2159	ctx->timestamp = now;
2160	perf_cgroup_set_timestamp(task, ctx);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2161	/*
2162	 * First go through the list and put on any pinned groups
2163	 * in order to give them the best chance of going on.
2164	 */
2165	if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2166		ctx_pinned_sched_in(ctx, cpuctx);
2167
2168	/* Then walk through the lower prio flexible groups */
2169	if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2170		ctx_flexible_sched_in(ctx, cpuctx);
2171}
2172
2173static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2174			     enum event_type_t event_type,
2175			     struct task_struct *task)
2176{
2177	struct perf_event_context *ctx = &cpuctx->ctx;
2178
2179	ctx_sched_in(ctx, cpuctx, event_type, task);
2180}
2181
2182static void perf_event_context_sched_in(struct perf_event_context *ctx,
2183					struct task_struct *task)
2184{
2185	struct perf_cpu_context *cpuctx;
2186
2187	cpuctx = __get_cpu_context(ctx);
2188	if (cpuctx->task_ctx == ctx)
2189		return;
2190
2191	perf_ctx_lock(cpuctx, ctx);
 
 
 
 
 
 
 
2192	perf_pmu_disable(ctx->pmu);
2193	/*
2194	 * We want to keep the following priority order:
2195	 * cpu pinned (that don't need to move), task pinned,
2196	 * cpu flexible, task flexible.
 
 
 
2197	 */
2198	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2199
2200	if (ctx->nr_events)
2201		cpuctx->task_ctx = ctx;
2202
2203	perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2204
2205	perf_pmu_enable(ctx->pmu);
2206	perf_ctx_unlock(cpuctx, ctx);
2207
2208	/*
2209	 * Since these rotations are per-cpu, we need to ensure the
2210	 * cpu-context we got scheduled on is actually rotating.
2211	 */
2212	perf_pmu_rotate_start(ctx->pmu);
2213}
2214
2215/*
2216 * When sampling the branck stack in system-wide, it may be necessary
2217 * to flush the stack on context switch. This happens when the branch
2218 * stack does not tag its entries with the pid of the current task.
2219 * Otherwise it becomes impossible to associate a branch entry with a
2220 * task. This ambiguity is more likely to appear when the branch stack
2221 * supports priv level filtering and the user sets it to monitor only
2222 * at the user level (which could be a useful measurement in system-wide
2223 * mode). In that case, the risk is high of having a branch stack with
2224 * branch from multiple tasks. Flushing may mean dropping the existing
2225 * entries or stashing them somewhere in the PMU specific code layer.
2226 *
2227 * This function provides the context switch callback to the lower code
2228 * layer. It is invoked ONLY when there is at least one system-wide context
2229 * with at least one active event using taken branch sampling.
2230 */
2231static void perf_branch_stack_sched_in(struct task_struct *prev,
2232				       struct task_struct *task)
2233{
2234	struct perf_cpu_context *cpuctx;
2235	struct pmu *pmu;
2236	unsigned long flags;
2237
2238	/* no need to flush branch stack if not changing task */
2239	if (prev == task)
2240		return;
2241
2242	local_irq_save(flags);
2243
2244	rcu_read_lock();
2245
2246	list_for_each_entry_rcu(pmu, &pmus, entry) {
2247		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2248
2249		/*
2250		 * check if the context has at least one
2251		 * event using PERF_SAMPLE_BRANCH_STACK
2252		 */
2253		if (cpuctx->ctx.nr_branch_stack > 0
2254		    && pmu->flush_branch_stack) {
2255
2256			pmu = cpuctx->ctx.pmu;
2257
2258			perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2259
2260			perf_pmu_disable(pmu);
2261
2262			pmu->flush_branch_stack();
2263
2264			perf_pmu_enable(pmu);
2265
2266			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2267		}
2268	}
2269
2270	rcu_read_unlock();
2271
2272	local_irq_restore(flags);
2273}
2274
2275/*
2276 * Called from scheduler to add the events of the current task
2277 * with interrupts disabled.
2278 *
2279 * We restore the event value and then enable it.
2280 *
2281 * This does not protect us against NMI, but enable()
2282 * sets the enabled bit in the control field of event _before_
2283 * accessing the event control register. If a NMI hits, then it will
2284 * keep the event running.
2285 */
2286void __perf_event_task_sched_in(struct task_struct *prev,
2287				struct task_struct *task)
2288{
2289	struct perf_event_context *ctx;
2290	int ctxn;
2291
 
 
 
 
 
 
 
 
 
 
2292	for_each_task_context_nr(ctxn) {
2293		ctx = task->perf_event_ctxp[ctxn];
2294		if (likely(!ctx))
2295			continue;
2296
2297		perf_event_context_sched_in(ctx, task);
2298	}
2299	/*
2300	 * if cgroup events exist on this CPU, then we need
2301	 * to check if we have to switch in PMU state.
2302	 * cgroup event are system-wide mode only
2303	 */
2304	if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2305		perf_cgroup_sched_in(prev, task);
2306
2307	/* check for system-wide branch_stack events */
2308	if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
2309		perf_branch_stack_sched_in(prev, task);
 
 
2310}
2311
2312static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2313{
2314	u64 frequency = event->attr.sample_freq;
2315	u64 sec = NSEC_PER_SEC;
2316	u64 divisor, dividend;
2317
2318	int count_fls, nsec_fls, frequency_fls, sec_fls;
2319
2320	count_fls = fls64(count);
2321	nsec_fls = fls64(nsec);
2322	frequency_fls = fls64(frequency);
2323	sec_fls = 30;
2324
2325	/*
2326	 * We got @count in @nsec, with a target of sample_freq HZ
2327	 * the target period becomes:
2328	 *
2329	 *             @count * 10^9
2330	 * period = -------------------
2331	 *          @nsec * sample_freq
2332	 *
2333	 */
2334
2335	/*
2336	 * Reduce accuracy by one bit such that @a and @b converge
2337	 * to a similar magnitude.
2338	 */
2339#define REDUCE_FLS(a, b)		\
2340do {					\
2341	if (a##_fls > b##_fls) {	\
2342		a >>= 1;		\
2343		a##_fls--;		\
2344	} else {			\
2345		b >>= 1;		\
2346		b##_fls--;		\
2347	}				\
2348} while (0)
2349
2350	/*
2351	 * Reduce accuracy until either term fits in a u64, then proceed with
2352	 * the other, so that finally we can do a u64/u64 division.
2353	 */
2354	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2355		REDUCE_FLS(nsec, frequency);
2356		REDUCE_FLS(sec, count);
2357	}
2358
2359	if (count_fls + sec_fls > 64) {
2360		divisor = nsec * frequency;
2361
2362		while (count_fls + sec_fls > 64) {
2363			REDUCE_FLS(count, sec);
2364			divisor >>= 1;
2365		}
2366
2367		dividend = count * sec;
2368	} else {
2369		dividend = count * sec;
2370
2371		while (nsec_fls + frequency_fls > 64) {
2372			REDUCE_FLS(nsec, frequency);
2373			dividend >>= 1;
2374		}
2375
2376		divisor = nsec * frequency;
2377	}
2378
2379	if (!divisor)
2380		return dividend;
2381
2382	return div64_u64(dividend, divisor);
2383}
2384
2385static DEFINE_PER_CPU(int, perf_throttled_count);
2386static DEFINE_PER_CPU(u64, perf_throttled_seq);
2387
2388static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2389{
2390	struct hw_perf_event *hwc = &event->hw;
2391	s64 period, sample_period;
2392	s64 delta;
2393
2394	period = perf_calculate_period(event, nsec, count);
2395
2396	delta = (s64)(period - hwc->sample_period);
2397	delta = (delta + 7) / 8; /* low pass filter */
2398
2399	sample_period = hwc->sample_period + delta;
2400
2401	if (!sample_period)
2402		sample_period = 1;
2403
2404	hwc->sample_period = sample_period;
2405
2406	if (local64_read(&hwc->period_left) > 8*sample_period) {
2407		if (disable)
2408			event->pmu->stop(event, PERF_EF_UPDATE);
2409
2410		local64_set(&hwc->period_left, 0);
2411
2412		if (disable)
2413			event->pmu->start(event, PERF_EF_RELOAD);
2414	}
2415}
2416
2417/*
2418 * combine freq adjustment with unthrottling to avoid two passes over the
2419 * events. At the same time, make sure, having freq events does not change
2420 * the rate of unthrottling as that would introduce bias.
2421 */
2422static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2423					   int needs_unthr)
2424{
2425	struct perf_event *event;
2426	struct hw_perf_event *hwc;
2427	u64 now, period = TICK_NSEC;
2428	s64 delta;
2429
2430	/*
2431	 * only need to iterate over all events iff:
2432	 * - context have events in frequency mode (needs freq adjust)
2433	 * - there are events to unthrottle on this cpu
2434	 */
2435	if (!(ctx->nr_freq || needs_unthr))
2436		return;
2437
2438	raw_spin_lock(&ctx->lock);
2439	perf_pmu_disable(ctx->pmu);
2440
2441	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2442		if (event->state != PERF_EVENT_STATE_ACTIVE)
2443			continue;
2444
2445		if (!event_filter_match(event))
2446			continue;
2447
 
 
2448		hwc = &event->hw;
2449
2450		if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
2451			hwc->interrupts = 0;
2452			perf_log_throttle(event, 1);
2453			event->pmu->start(event, 0);
2454		}
2455
2456		if (!event->attr.freq || !event->attr.sample_freq)
2457			continue;
2458
2459		/*
2460		 * stop the event and update event->count
2461		 */
2462		event->pmu->stop(event, PERF_EF_UPDATE);
2463
2464		now = local64_read(&event->count);
2465		delta = now - hwc->freq_count_stamp;
2466		hwc->freq_count_stamp = now;
2467
2468		/*
2469		 * restart the event
2470		 * reload only if value has changed
2471		 * we have stopped the event so tell that
2472		 * to perf_adjust_period() to avoid stopping it
2473		 * twice.
2474		 */
2475		if (delta > 0)
2476			perf_adjust_period(event, period, delta, false);
2477
2478		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
 
 
2479	}
2480
2481	perf_pmu_enable(ctx->pmu);
2482	raw_spin_unlock(&ctx->lock);
2483}
2484
2485/*
2486 * Round-robin a context's events:
2487 */
2488static void rotate_ctx(struct perf_event_context *ctx)
2489{
2490	/*
2491	 * Rotate the first entry last of non-pinned groups. Rotation might be
2492	 * disabled by the inheritance code.
2493	 */
2494	if (!ctx->rotate_disable)
2495		list_rotate_left(&ctx->flexible_groups);
 
 
 
2496}
2497
2498/*
2499 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2500 * because they're strictly cpu affine and rotate_start is called with IRQs
2501 * disabled, while rotate_context is called from IRQ context.
2502 */
2503static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2504{
2505	struct perf_event_context *ctx = NULL;
2506	int rotate = 0, remove = 1;
2507
2508	if (cpuctx->ctx.nr_events) {
2509		remove = 0;
2510		if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2511			rotate = 1;
 
 
 
 
2512	}
2513
2514	ctx = cpuctx->task_ctx;
2515	if (ctx && ctx->nr_events) {
2516		remove = 0;
2517		if (ctx->nr_events != ctx->nr_active)
2518			rotate = 1;
2519	}
2520
2521	if (!rotate)
2522		goto done;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2523
2524	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2525	perf_pmu_disable(cpuctx->ctx.pmu);
2526
2527	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2528	if (ctx)
2529		ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2530
2531	rotate_ctx(&cpuctx->ctx);
2532	if (ctx)
2533		rotate_ctx(ctx);
 
 
 
 
 
 
 
 
 
 
 
2534
2535	perf_event_sched_in(cpuctx, ctx, current);
2536
2537	perf_pmu_enable(cpuctx->ctx.pmu);
2538	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2539done:
2540	if (remove)
2541		list_del_init(&cpuctx->rotation_list);
2542}
2543
2544void perf_event_task_tick(void)
2545{
2546	struct list_head *head = &__get_cpu_var(rotation_list);
2547	struct perf_cpu_context *cpuctx, *tmp;
2548	struct perf_event_context *ctx;
2549	int throttled;
2550
2551	WARN_ON(!irqs_disabled());
2552
2553	__this_cpu_inc(perf_throttled_seq);
2554	throttled = __this_cpu_xchg(perf_throttled_count, 0);
 
2555
2556	list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2557		ctx = &cpuctx->ctx;
2558		perf_adjust_freq_unthr_context(ctx, throttled);
2559
2560		ctx = cpuctx->task_ctx;
2561		if (ctx)
2562			perf_adjust_freq_unthr_context(ctx, throttled);
2563
2564		if (cpuctx->jiffies_interval == 1 ||
2565				!(jiffies % cpuctx->jiffies_interval))
2566			perf_rotate_context(cpuctx);
2567	}
2568}
2569
2570static int event_enable_on_exec(struct perf_event *event,
2571				struct perf_event_context *ctx)
2572{
2573	if (!event->attr.enable_on_exec)
2574		return 0;
2575
2576	event->attr.enable_on_exec = 0;
2577	if (event->state >= PERF_EVENT_STATE_INACTIVE)
2578		return 0;
2579
2580	__perf_event_mark_enabled(event);
2581
2582	return 1;
2583}
2584
2585/*
2586 * Enable all of a task's events that have been marked enable-on-exec.
2587 * This expects task == current.
2588 */
2589static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2590{
 
 
 
2591	struct perf_event *event;
2592	unsigned long flags;
2593	int enabled = 0;
2594	int ret;
2595
2596	local_irq_save(flags);
 
2597	if (!ctx || !ctx->nr_events)
2598		goto out;
2599
2600	/*
2601	 * We must ctxsw out cgroup events to avoid conflict
2602	 * when invoking perf_task_event_sched_in() later on
2603	 * in this function. Otherwise we end up trying to
2604	 * ctxswin cgroup events which are already scheduled
2605	 * in.
2606	 */
2607	perf_cgroup_sched_out(current, NULL);
2608
2609	raw_spin_lock(&ctx->lock);
2610	task_ctx_sched_out(ctx);
2611
2612	list_for_each_entry(event, &ctx->event_list, event_entry) {
2613		ret = event_enable_on_exec(event, ctx);
2614		if (ret)
2615			enabled = 1;
2616	}
2617
2618	/*
2619	 * Unclone this context if we enabled any event.
2620	 */
2621	if (enabled)
2622		unclone_ctx(ctx);
2623
2624	raw_spin_unlock(&ctx->lock);
 
 
 
2625
2626	/*
2627	 * Also calls ctxswin for cgroup events, if any:
2628	 */
2629	perf_event_context_sched_in(ctx, ctx->task);
2630out:
2631	local_irq_restore(flags);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2632}
2633
2634/*
2635 * Cross CPU call to read the hardware event
2636 */
2637static void __perf_event_read(void *info)
2638{
2639	struct perf_event *event = info;
 
2640	struct perf_event_context *ctx = event->ctx;
2641	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
 
2642
2643	/*
2644	 * If this is a task context, we need to check whether it is
2645	 * the current task context of this cpu.  If not it has been
2646	 * scheduled out before the smp call arrived.  In that case
2647	 * event->count would have been updated to a recent sample
2648	 * when the event was scheduled out.
2649	 */
2650	if (ctx->task && cpuctx->task_ctx != ctx)
2651		return;
2652
2653	raw_spin_lock(&ctx->lock);
2654	if (ctx->is_active) {
2655		update_context_time(ctx);
2656		update_cgrp_time_from_event(event);
2657	}
2658	update_event_times(event);
2659	if (event->state == PERF_EVENT_STATE_ACTIVE)
2660		event->pmu->read(event);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2661	raw_spin_unlock(&ctx->lock);
2662}
2663
2664static inline u64 perf_event_count(struct perf_event *event)
2665{
2666	return local64_read(&event->count) + atomic64_read(&event->child_count);
2667}
2668
2669static u64 perf_event_read(struct perf_event *event)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2670{
 
 
 
2671	/*
2672	 * If event is enabled and currently active on a CPU, update the
2673	 * value in the event structure:
2674	 */
2675	if (event->state == PERF_EVENT_STATE_ACTIVE) {
2676		smp_call_function_single(event->oncpu,
2677					 __perf_event_read, event, 1);
2678	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2679		struct perf_event_context *ctx = event->ctx;
2680		unsigned long flags;
2681
2682		raw_spin_lock_irqsave(&ctx->lock, flags);
 
 
 
 
 
 
2683		/*
2684		 * may read while context is not active
2685		 * (e.g., thread is blocked), in that case
2686		 * we cannot update context time
2687		 */
2688		if (ctx->is_active) {
2689			update_context_time(ctx);
2690			update_cgrp_time_from_event(event);
2691		}
2692		update_event_times(event);
 
 
 
2693		raw_spin_unlock_irqrestore(&ctx->lock, flags);
2694	}
2695
2696	return perf_event_count(event);
2697}
2698
2699/*
2700 * Initialize the perf_event context in a task_struct:
2701 */
2702static void __perf_event_init_context(struct perf_event_context *ctx)
2703{
2704	raw_spin_lock_init(&ctx->lock);
2705	mutex_init(&ctx->mutex);
2706	INIT_LIST_HEAD(&ctx->pinned_groups);
2707	INIT_LIST_HEAD(&ctx->flexible_groups);
 
2708	INIT_LIST_HEAD(&ctx->event_list);
2709	atomic_set(&ctx->refcount, 1);
 
 
2710}
2711
2712static struct perf_event_context *
2713alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2714{
2715	struct perf_event_context *ctx;
2716
2717	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2718	if (!ctx)
2719		return NULL;
2720
2721	__perf_event_init_context(ctx);
2722	if (task) {
2723		ctx->task = task;
2724		get_task_struct(task);
2725	}
2726	ctx->pmu = pmu;
2727
2728	return ctx;
2729}
2730
2731static struct task_struct *
2732find_lively_task_by_vpid(pid_t vpid)
2733{
2734	struct task_struct *task;
2735	int err;
2736
2737	rcu_read_lock();
2738	if (!vpid)
2739		task = current;
2740	else
2741		task = find_task_by_vpid(vpid);
2742	if (task)
2743		get_task_struct(task);
2744	rcu_read_unlock();
2745
2746	if (!task)
2747		return ERR_PTR(-ESRCH);
2748
2749	/* Reuse ptrace permission checks for now. */
2750	err = -EACCES;
2751	if (!ptrace_may_access(task, PTRACE_MODE_READ))
2752		goto errout;
2753
2754	return task;
2755errout:
2756	put_task_struct(task);
2757	return ERR_PTR(err);
2758
2759}
2760
2761/*
2762 * Returns a matching context with refcount and pincount.
2763 */
2764static struct perf_event_context *
2765find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
 
2766{
2767	struct perf_event_context *ctx;
2768	struct perf_cpu_context *cpuctx;
 
2769	unsigned long flags;
2770	int ctxn, err;
 
2771
2772	if (!task) {
2773		/* Must be root to operate on a CPU event: */
2774		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2775			return ERR_PTR(-EACCES);
2776
2777		/*
2778		 * We could be clever and allow to attach a event to an
2779		 * offline CPU and activate it when the CPU comes up, but
2780		 * that's for later.
2781		 */
2782		if (!cpu_online(cpu))
2783			return ERR_PTR(-ENODEV);
2784
2785		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2786		ctx = &cpuctx->ctx;
2787		get_ctx(ctx);
2788		++ctx->pin_count;
2789
2790		return ctx;
2791	}
2792
2793	err = -EINVAL;
2794	ctxn = pmu->task_ctx_nr;
2795	if (ctxn < 0)
2796		goto errout;
2797
 
 
 
 
 
 
 
 
2798retry:
2799	ctx = perf_lock_task_context(task, ctxn, &flags);
2800	if (ctx) {
2801		unclone_ctx(ctx);
2802		++ctx->pin_count;
 
 
 
 
 
2803		raw_spin_unlock_irqrestore(&ctx->lock, flags);
 
 
 
2804	} else {
2805		ctx = alloc_perf_context(pmu, task);
2806		err = -ENOMEM;
2807		if (!ctx)
2808			goto errout;
2809
 
 
 
 
 
2810		err = 0;
2811		mutex_lock(&task->perf_event_mutex);
2812		/*
2813		 * If it has already passed perf_event_exit_task().
2814		 * we must see PF_EXITING, it takes this mutex too.
2815		 */
2816		if (task->flags & PF_EXITING)
2817			err = -ESRCH;
2818		else if (task->perf_event_ctxp[ctxn])
2819			err = -EAGAIN;
2820		else {
2821			get_ctx(ctx);
2822			++ctx->pin_count;
2823			rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2824		}
2825		mutex_unlock(&task->perf_event_mutex);
2826
2827		if (unlikely(err)) {
2828			put_ctx(ctx);
2829
2830			if (err == -EAGAIN)
2831				goto retry;
2832			goto errout;
2833		}
2834	}
2835
 
2836	return ctx;
2837
2838errout:
 
2839	return ERR_PTR(err);
2840}
2841
2842static void perf_event_free_filter(struct perf_event *event);
 
2843
2844static void free_event_rcu(struct rcu_head *head)
2845{
2846	struct perf_event *event;
2847
2848	event = container_of(head, struct perf_event, rcu_head);
2849	if (event->ns)
2850		put_pid_ns(event->ns);
2851	perf_event_free_filter(event);
2852	kfree(event);
2853}
2854
2855static void ring_buffer_put(struct ring_buffer *rb);
 
2856
2857static void free_event(struct perf_event *event)
2858{
2859	irq_work_sync(&event->pending);
2860
2861	if (!event->parent) {
2862		if (event->attach_state & PERF_ATTACH_TASK)
2863			static_key_slow_dec_deferred(&perf_sched_events);
2864		if (event->attr.mmap || event->attr.mmap_data)
2865			atomic_dec(&nr_mmap_events);
2866		if (event->attr.comm)
2867			atomic_dec(&nr_comm_events);
2868		if (event->attr.task)
2869			atomic_dec(&nr_task_events);
2870		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2871			put_callchain_buffers();
2872		if (is_cgroup_event(event)) {
2873			atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
2874			static_key_slow_dec_deferred(&perf_sched_events);
2875		}
2876
2877		if (has_branch_stack(event)) {
2878			static_key_slow_dec_deferred(&perf_sched_events);
2879			/* is system-wide event */
2880			if (!(event->attach_state & PERF_ATTACH_TASK))
2881				atomic_dec(&per_cpu(perf_branch_stack_events,
2882						    event->cpu));
2883		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2884	}
2885
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2886	if (event->rb) {
2887		ring_buffer_put(event->rb);
2888		event->rb = NULL;
 
 
 
 
 
 
 
2889	}
2890
2891	if (is_cgroup_event(event))
2892		perf_detach_cgroup(event);
2893
 
 
 
 
 
 
 
 
 
2894	if (event->destroy)
2895		event->destroy(event);
2896
 
 
 
 
 
 
 
 
 
 
 
2897	if (event->ctx)
2898		put_ctx(event->ctx);
2899
 
 
 
2900	call_rcu(&event->rcu_head, free_event_rcu);
2901}
2902
2903int perf_event_release_kernel(struct perf_event *event)
 
 
 
 
2904{
2905	struct perf_event_context *ctx = event->ctx;
2906
2907	WARN_ON_ONCE(ctx->parent_ctx);
2908	/*
2909	 * There are two ways this annotation is useful:
2910	 *
2911	 *  1) there is a lock recursion from perf_event_exit_task
2912	 *     see the comment there.
2913	 *
2914	 *  2) there is a lock-inversion with mmap_sem through
2915	 *     perf_event_read_group(), which takes faults while
2916	 *     holding ctx->mutex, however this is called after
2917	 *     the last filedesc died, so there is no possibility
2918	 *     to trigger the AB-BA case.
2919	 */
2920	mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2921	raw_spin_lock_irq(&ctx->lock);
2922	perf_group_detach(event);
2923	raw_spin_unlock_irq(&ctx->lock);
2924	perf_remove_from_context(event);
2925	mutex_unlock(&ctx->mutex);
2926
2927	free_event(event);
2928
2929	return 0;
2930}
2931EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2932
2933/*
2934 * Called when the last reference to the file is gone.
2935 */
2936static void put_event(struct perf_event *event)
2937{
2938	struct task_struct *owner;
2939
2940	if (!atomic_long_dec_and_test(&event->refcount))
2941		return;
2942
2943	rcu_read_lock();
2944	owner = ACCESS_ONCE(event->owner);
2945	/*
2946	 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
2947	 * !owner it means the list deletion is complete and we can indeed
2948	 * free this event, otherwise we need to serialize on
2949	 * owner->perf_event_mutex.
2950	 */
2951	smp_read_barrier_depends();
2952	if (owner) {
2953		/*
2954		 * Since delayed_put_task_struct() also drops the last
2955		 * task reference we can safely take a new reference
2956		 * while holding the rcu_read_lock().
2957		 */
2958		get_task_struct(owner);
2959	}
2960	rcu_read_unlock();
2961
2962	if (owner) {
2963		mutex_lock(&owner->perf_event_mutex);
 
 
 
 
 
 
 
 
 
2964		/*
2965		 * We have to re-check the event->owner field, if it is cleared
2966		 * we raced with perf_event_exit_task(), acquiring the mutex
2967		 * ensured they're done, and we can proceed with freeing the
2968		 * event.
2969		 */
2970		if (event->owner)
2971			list_del_init(&event->owner_entry);
 
 
2972		mutex_unlock(&owner->perf_event_mutex);
2973		put_task_struct(owner);
2974	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2975
2976	perf_event_release_kernel(event);
 
 
 
 
 
 
 
 
 
 
 
 
 
2977}
 
2978
 
 
 
2979static int perf_release(struct inode *inode, struct file *file)
2980{
2981	put_event(file->private_data);
2982	return 0;
2983}
2984
2985u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2986{
2987	struct perf_event *child;
2988	u64 total = 0;
2989
2990	*enabled = 0;
2991	*running = 0;
2992
2993	mutex_lock(&event->child_mutex);
2994	total += perf_event_read(event);
 
 
 
2995	*enabled += event->total_time_enabled +
2996			atomic64_read(&event->child_total_time_enabled);
2997	*running += event->total_time_running +
2998			atomic64_read(&event->child_total_time_running);
2999
3000	list_for_each_entry(child, &event->child_list, child_list) {
3001		total += perf_event_read(child);
 
3002		*enabled += child->total_time_enabled;
3003		*running += child->total_time_running;
3004	}
3005	mutex_unlock(&event->child_mutex);
3006
3007	return total;
3008}
 
 
 
 
 
 
 
 
 
 
 
 
3009EXPORT_SYMBOL_GPL(perf_event_read_value);
3010
3011static int perf_event_read_group(struct perf_event *event,
3012				   u64 read_format, char __user *buf)
3013{
3014	struct perf_event *leader = event->group_leader, *sub;
3015	int n = 0, size = 0, ret = -EFAULT;
3016	struct perf_event_context *ctx = leader->ctx;
3017	u64 values[5];
3018	u64 count, enabled, running;
 
 
3019
3020	mutex_lock(&ctx->mutex);
3021	count = perf_event_read_value(leader, &enabled, &running);
 
3022
3023	values[n++] = 1 + leader->nr_siblings;
3024	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3025		values[n++] = enabled;
3026	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3027		values[n++] = running;
3028	values[n++] = count;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3029	if (read_format & PERF_FORMAT_ID)
3030		values[n++] = primary_event_id(leader);
3031
3032	size = n * sizeof(u64);
 
 
 
 
3033
3034	if (copy_to_user(buf, values, size))
3035		goto unlock;
 
3036
3037	ret = size;
 
 
 
 
 
 
3038
3039	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3040		n = 0;
3041
3042		values[n++] = perf_event_read_value(sub, &enabled, &running);
3043		if (read_format & PERF_FORMAT_ID)
3044			values[n++] = primary_event_id(sub);
3045
3046		size = n * sizeof(u64);
3047
3048		if (copy_to_user(buf + ret, values, size)) {
3049			ret = -EFAULT;
3050			goto unlock;
3051		}
 
3052
3053		ret += size;
 
 
 
 
 
 
 
3054	}
3055unlock:
3056	mutex_unlock(&ctx->mutex);
3057
 
 
 
 
 
 
 
 
 
 
 
3058	return ret;
3059}
3060
3061static int perf_event_read_one(struct perf_event *event,
3062				 u64 read_format, char __user *buf)
3063{
3064	u64 enabled, running;
3065	u64 values[4];
3066	int n = 0;
3067
3068	values[n++] = perf_event_read_value(event, &enabled, &running);
3069	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3070		values[n++] = enabled;
3071	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3072		values[n++] = running;
3073	if (read_format & PERF_FORMAT_ID)
3074		values[n++] = primary_event_id(event);
3075
3076	if (copy_to_user(buf, values, n * sizeof(u64)))
3077		return -EFAULT;
3078
3079	return n * sizeof(u64);
3080}
3081
 
 
 
 
 
 
 
 
 
 
 
 
 
3082/*
3083 * Read the performance event - simple non blocking version for now
3084 */
3085static ssize_t
3086perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3087{
3088	u64 read_format = event->attr.read_format;
3089	int ret;
3090
3091	/*
3092	 * Return end-of-file for a read on a event that is in
3093	 * error state (i.e. because it was pinned but it couldn't be
3094	 * scheduled on to the CPU at some point).
3095	 */
3096	if (event->state == PERF_EVENT_STATE_ERROR)
3097		return 0;
3098
3099	if (count < event->read_size)
3100		return -ENOSPC;
3101
3102	WARN_ON_ONCE(event->ctx->parent_ctx);
3103	if (read_format & PERF_FORMAT_GROUP)
3104		ret = perf_event_read_group(event, read_format, buf);
3105	else
3106		ret = perf_event_read_one(event, read_format, buf);
3107
3108	return ret;
3109}
3110
3111static ssize_t
3112perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3113{
3114	struct perf_event *event = file->private_data;
 
 
 
 
 
 
3115
3116	return perf_read_hw(event, buf, count);
3117}
3118
3119static unsigned int perf_poll(struct file *file, poll_table *wait)
3120{
3121	struct perf_event *event = file->private_data;
3122	struct ring_buffer *rb;
3123	unsigned int events = POLL_HUP;
 
 
 
 
 
3124
3125	/*
3126	 * Race between perf_event_set_output() and perf_poll(): perf_poll()
3127	 * grabs the rb reference but perf_event_set_output() overrides it.
3128	 * Here is the timeline for two threads T1, T2:
3129	 * t0: T1, rb = rcu_dereference(event->rb)
3130	 * t1: T2, old_rb = event->rb
3131	 * t2: T2, event->rb = new rb
3132	 * t3: T2, ring_buffer_detach(old_rb)
3133	 * t4: T1, ring_buffer_attach(rb1)
3134	 * t5: T1, poll_wait(event->waitq)
3135	 *
3136	 * To avoid this problem, we grab mmap_mutex in perf_poll()
3137	 * thereby ensuring that the assignment of the new ring buffer
3138	 * and the detachment of the old buffer appear atomic to perf_poll()
3139	 */
3140	mutex_lock(&event->mmap_mutex);
3141
3142	rcu_read_lock();
3143	rb = rcu_dereference(event->rb);
3144	if (rb) {
3145		ring_buffer_attach(event, rb);
3146		events = atomic_xchg(&rb->poll, 0);
3147	}
3148	rcu_read_unlock();
3149
3150	mutex_unlock(&event->mmap_mutex);
3151
3152	poll_wait(file, &event->waitq, wait);
3153
3154	return events;
3155}
3156
3157static void perf_event_reset(struct perf_event *event)
3158{
3159	(void)perf_event_read(event);
3160	local64_set(&event->count, 0);
3161	perf_event_update_userpage(event);
3162}
3163
3164/*
3165 * Holding the top-level event's child_mutex means that any
3166 * descendant process that has inherited this event will block
3167 * in sync_child_event if it goes to exit, thus satisfying the
3168 * task existence requirements of perf_event_enable/disable.
3169 */
3170static void perf_event_for_each_child(struct perf_event *event,
3171					void (*func)(struct perf_event *))
3172{
3173	struct perf_event *child;
3174
3175	WARN_ON_ONCE(event->ctx->parent_ctx);
 
3176	mutex_lock(&event->child_mutex);
3177	func(event);
3178	list_for_each_entry(child, &event->child_list, child_list)
3179		func(child);
3180	mutex_unlock(&event->child_mutex);
3181}
3182
3183static void perf_event_for_each(struct perf_event *event,
3184				  void (*func)(struct perf_event *))
3185{
3186	struct perf_event_context *ctx = event->ctx;
3187	struct perf_event *sibling;
3188
3189	WARN_ON_ONCE(ctx->parent_ctx);
3190	mutex_lock(&ctx->mutex);
3191	event = event->group_leader;
3192
3193	perf_event_for_each_child(event, func);
3194	list_for_each_entry(sibling, &event->sibling_list, group_entry)
3195		perf_event_for_each_child(sibling, func);
3196	mutex_unlock(&ctx->mutex);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3197}
3198
3199static int perf_event_period(struct perf_event *event, u64 __user *arg)
3200{
3201	struct perf_event_context *ctx = event->ctx;
3202	int ret = 0;
3203	u64 value;
3204
3205	if (!is_sampling_event(event))
3206		return -EINVAL;
3207
3208	if (copy_from_user(&value, arg, sizeof(value)))
3209		return -EFAULT;
3210
3211	if (!value)
3212		return -EINVAL;
3213
3214	raw_spin_lock_irq(&ctx->lock);
3215	if (event->attr.freq) {
3216		if (value > sysctl_perf_event_sample_rate) {
3217			ret = -EINVAL;
3218			goto unlock;
3219		}
3220
3221		event->attr.sample_freq = value;
3222	} else {
3223		event->attr.sample_period = value;
3224		event->hw.sample_period = value;
3225	}
3226unlock:
3227	raw_spin_unlock_irq(&ctx->lock);
3228
3229	return ret;
 
 
 
 
 
3230}
3231
3232static const struct file_operations perf_fops;
3233
3234static struct file *perf_fget_light(int fd, int *fput_needed)
3235{
3236	struct file *file;
3237
3238	file = fget_light(fd, fput_needed);
3239	if (!file)
3240		return ERR_PTR(-EBADF);
3241
3242	if (file->f_op != &perf_fops) {
3243		fput_light(file, *fput_needed);
3244		*fput_needed = 0;
3245		return ERR_PTR(-EBADF);
3246	}
3247
3248	return file;
3249}
3250
3251static int perf_event_set_output(struct perf_event *event,
3252				 struct perf_event *output_event);
3253static int perf_event_set_filter(struct perf_event *event, void __user *arg);
 
 
 
3254
3255static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3256{
3257	struct perf_event *event = file->private_data;
3258	void (*func)(struct perf_event *);
3259	u32 flags = arg;
3260
3261	switch (cmd) {
3262	case PERF_EVENT_IOC_ENABLE:
3263		func = perf_event_enable;
3264		break;
3265	case PERF_EVENT_IOC_DISABLE:
3266		func = perf_event_disable;
3267		break;
3268	case PERF_EVENT_IOC_RESET:
3269		func = perf_event_reset;
3270		break;
3271
3272	case PERF_EVENT_IOC_REFRESH:
3273		return perf_event_refresh(event, arg);
3274
3275	case PERF_EVENT_IOC_PERIOD:
3276		return perf_event_period(event, (u64 __user *)arg);
3277
 
 
 
 
 
 
 
 
 
3278	case PERF_EVENT_IOC_SET_OUTPUT:
3279	{
3280		struct file *output_file = NULL;
3281		struct perf_event *output_event = NULL;
3282		int fput_needed = 0;
3283		int ret;
3284
3285		if (arg != -1) {
3286			output_file = perf_fget_light(arg, &fput_needed);
3287			if (IS_ERR(output_file))
3288				return PTR_ERR(output_file);
3289			output_event = output_file->private_data;
 
 
 
 
 
 
3290		}
3291
3292		ret = perf_event_set_output(event, output_event);
3293		if (output_event)
3294			fput_light(output_file, fput_needed);
3295
3296		return ret;
3297	}
3298
3299	case PERF_EVENT_IOC_SET_FILTER:
3300		return perf_event_set_filter(event, (void __user *)arg);
3301
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3302	default:
3303		return -ENOTTY;
3304	}
3305
3306	if (flags & PERF_IOC_FLAG_GROUP)
3307		perf_event_for_each(event, func);
3308	else
3309		perf_event_for_each_child(event, func);
3310
3311	return 0;
3312}
3313
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3314int perf_event_task_enable(void)
3315{
 
3316	struct perf_event *event;
3317
3318	mutex_lock(&current->perf_event_mutex);
3319	list_for_each_entry(event, &current->perf_event_list, owner_entry)
3320		perf_event_for_each_child(event, perf_event_enable);
 
 
 
3321	mutex_unlock(&current->perf_event_mutex);
3322
3323	return 0;
3324}
3325
3326int perf_event_task_disable(void)
3327{
 
3328	struct perf_event *event;
3329
3330	mutex_lock(&current->perf_event_mutex);
3331	list_for_each_entry(event, &current->perf_event_list, owner_entry)
3332		perf_event_for_each_child(event, perf_event_disable);
 
 
 
3333	mutex_unlock(&current->perf_event_mutex);
3334
3335	return 0;
3336}
3337
3338static int perf_event_index(struct perf_event *event)
3339{
3340	if (event->hw.state & PERF_HES_STOPPED)
3341		return 0;
3342
3343	if (event->state != PERF_EVENT_STATE_ACTIVE)
3344		return 0;
3345
3346	return event->pmu->event_idx(event);
3347}
3348
3349static void calc_timer_values(struct perf_event *event,
3350				u64 *now,
3351				u64 *enabled,
3352				u64 *running)
3353{
3354	u64 ctx_time;
3355
3356	*now = perf_clock();
3357	ctx_time = event->shadow_ctx_time + *now;
3358	*enabled = ctx_time - event->tstamp_enabled;
3359	*running = ctx_time - event->tstamp_running;
3360}
3361
3362void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3363{
3364}
3365
3366/*
3367 * Callers need to ensure there can be no nesting of this function, otherwise
3368 * the seqlock logic goes bad. We can not serialize this because the arch
3369 * code calls this from NMI context.
3370 */
3371void perf_event_update_userpage(struct perf_event *event)
3372{
3373	struct perf_event_mmap_page *userpg;
3374	struct ring_buffer *rb;
3375	u64 enabled, running, now;
3376
3377	rcu_read_lock();
 
 
 
 
3378	/*
3379	 * compute total_time_enabled, total_time_running
3380	 * based on snapshot values taken when the event
3381	 * was last scheduled in.
3382	 *
3383	 * we cannot simply called update_context_time()
3384	 * because of locking issue as we can be called in
3385	 * NMI context
3386	 */
3387	calc_timer_values(event, &now, &enabled, &running);
3388	rb = rcu_dereference(event->rb);
3389	if (!rb)
3390		goto unlock;
3391
3392	userpg = rb->user_page;
3393
3394	/*
3395	 * Disable preemption so as to not let the corresponding user-space
3396	 * spin too long if we get preempted.
3397	 */
3398	preempt_disable();
3399	++userpg->lock;
3400	barrier();
3401	userpg->index = perf_event_index(event);
3402	userpg->offset = perf_event_count(event);
3403	if (userpg->index)
3404		userpg->offset -= local64_read(&event->hw.prev_count);
3405
3406	userpg->time_enabled = enabled +
3407			atomic64_read(&event->child_total_time_enabled);
3408
3409	userpg->time_running = running +
3410			atomic64_read(&event->child_total_time_running);
3411
3412	arch_perf_update_userpage(userpg, now);
3413
3414	barrier();
3415	++userpg->lock;
3416	preempt_enable();
3417unlock:
3418	rcu_read_unlock();
3419}
 
3420
3421static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3422{
3423	struct perf_event *event = vma->vm_file->private_data;
3424	struct ring_buffer *rb;
3425	int ret = VM_FAULT_SIGBUS;
3426
3427	if (vmf->flags & FAULT_FLAG_MKWRITE) {
3428		if (vmf->pgoff == 0)
3429			ret = 0;
3430		return ret;
3431	}
3432
3433	rcu_read_lock();
3434	rb = rcu_dereference(event->rb);
3435	if (!rb)
3436		goto unlock;
3437
3438	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3439		goto unlock;
3440
3441	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
3442	if (!vmf->page)
3443		goto unlock;
3444
3445	get_page(vmf->page);
3446	vmf->page->mapping = vma->vm_file->f_mapping;
3447	vmf->page->index   = vmf->pgoff;
3448
3449	ret = 0;
3450unlock:
3451	rcu_read_unlock();
3452
3453	return ret;
3454}
3455
3456static void ring_buffer_attach(struct perf_event *event,
3457			       struct ring_buffer *rb)
3458{
 
3459	unsigned long flags;
3460
3461	if (!list_empty(&event->rb_entry))
3462		return;
 
 
 
 
3463
3464	spin_lock_irqsave(&rb->event_lock, flags);
3465	if (!list_empty(&event->rb_entry))
3466		goto unlock;
 
3467
3468	list_add(&event->rb_entry, &rb->event_list);
3469unlock:
3470	spin_unlock_irqrestore(&rb->event_lock, flags);
3471}
3472
3473static void ring_buffer_detach(struct perf_event *event,
3474			       struct ring_buffer *rb)
3475{
3476	unsigned long flags;
 
3477
3478	if (list_empty(&event->rb_entry))
3479		return;
 
 
3480
3481	spin_lock_irqsave(&rb->event_lock, flags);
3482	list_del_init(&event->rb_entry);
3483	wake_up_all(&event->waitq);
3484	spin_unlock_irqrestore(&rb->event_lock, flags);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3485}
3486
3487static void ring_buffer_wakeup(struct perf_event *event)
3488{
3489	struct ring_buffer *rb;
3490
3491	rcu_read_lock();
3492	rb = rcu_dereference(event->rb);
3493	if (!rb)
3494		goto unlock;
3495
3496	list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3497		wake_up_all(&event->waitq);
3498
3499unlock:
3500	rcu_read_unlock();
3501}
3502
3503static void rb_free_rcu(struct rcu_head *rcu_head)
3504{
3505	struct ring_buffer *rb;
3506
3507	rb = container_of(rcu_head, struct ring_buffer, rcu_head);
3508	rb_free(rb);
3509}
3510
3511static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3512{
3513	struct ring_buffer *rb;
3514
3515	rcu_read_lock();
3516	rb = rcu_dereference(event->rb);
3517	if (rb) {
3518		if (!atomic_inc_not_zero(&rb->refcount))
3519			rb = NULL;
3520	}
3521	rcu_read_unlock();
3522
3523	return rb;
3524}
3525
3526static void ring_buffer_put(struct ring_buffer *rb)
3527{
3528	struct perf_event *event, *n;
3529	unsigned long flags;
3530
3531	if (!atomic_dec_and_test(&rb->refcount))
3532		return;
3533
3534	spin_lock_irqsave(&rb->event_lock, flags);
3535	list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) {
3536		list_del_init(&event->rb_entry);
3537		wake_up_all(&event->waitq);
3538	}
3539	spin_unlock_irqrestore(&rb->event_lock, flags);
3540
3541	call_rcu(&rb->rcu_head, rb_free_rcu);
3542}
3543
3544static void perf_mmap_open(struct vm_area_struct *vma)
3545{
3546	struct perf_event *event = vma->vm_file->private_data;
3547
3548	atomic_inc(&event->mmap_count);
 
 
 
 
 
 
 
3549}
3550
 
 
 
 
 
 
 
 
 
 
3551static void perf_mmap_close(struct vm_area_struct *vma)
3552{
3553	struct perf_event *event = vma->vm_file->private_data;
3554
3555	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3556		unsigned long size = perf_data_size(event->rb);
3557		struct user_struct *user = event->mmap_user;
3558		struct ring_buffer *rb = event->rb;
3559
3560		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3561		vma->vm_mm->pinned_vm -= event->mmap_locked;
3562		rcu_assign_pointer(event->rb, NULL);
3563		ring_buffer_detach(event, rb);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3564		mutex_unlock(&event->mmap_mutex);
 
 
 
 
 
 
3565
3566		ring_buffer_put(rb);
3567		free_uid(user);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3568	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3569}
3570
3571static const struct vm_operations_struct perf_mmap_vmops = {
3572	.open		= perf_mmap_open,
3573	.close		= perf_mmap_close,
3574	.fault		= perf_mmap_fault,
3575	.page_mkwrite	= perf_mmap_fault,
3576};
3577
3578static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3579{
3580	struct perf_event *event = file->private_data;
3581	unsigned long user_locked, user_lock_limit;
3582	struct user_struct *user = current_user();
3583	unsigned long locked, lock_limit;
3584	struct ring_buffer *rb;
3585	unsigned long vma_size;
3586	unsigned long nr_pages;
3587	long user_extra, extra;
3588	int ret = 0, flags = 0;
3589
3590	/*
3591	 * Don't allow mmap() of inherited per-task counters. This would
3592	 * create a performance issue due to all children writing to the
3593	 * same rb.
3594	 */
3595	if (event->cpu == -1 && event->attr.inherit)
3596		return -EINVAL;
3597
3598	if (!(vma->vm_flags & VM_SHARED))
3599		return -EINVAL;
3600
3601	vma_size = vma->vm_end - vma->vm_start;
3602	nr_pages = (vma_size / PAGE_SIZE) - 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3603
3604	/*
3605	 * If we have rb pages ensure they're a power-of-two number, so we
3606	 * can do bitmasks instead of modulo.
3607	 */
3608	if (nr_pages != 0 && !is_power_of_2(nr_pages))
3609		return -EINVAL;
3610
3611	if (vma_size != PAGE_SIZE * (1 + nr_pages))
3612		return -EINVAL;
3613
3614	if (vma->vm_pgoff != 0)
3615		return -EINVAL;
3616
3617	WARN_ON_ONCE(event->ctx->parent_ctx);
 
3618	mutex_lock(&event->mmap_mutex);
3619	if (event->rb) {
3620		if (event->rb->nr_pages == nr_pages)
3621			atomic_inc(&event->rb->refcount);
3622		else
3623			ret = -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
 
 
3624		goto unlock;
3625	}
3626
3627	user_extra = nr_pages + 1;
 
 
3628	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3629
3630	/*
3631	 * Increase the limit linearly with more CPUs:
3632	 */
3633	user_lock_limit *= num_online_cpus();
3634
3635	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3636
3637	extra = 0;
3638	if (user_locked > user_lock_limit)
 
 
 
 
 
 
 
 
 
3639		extra = user_locked - user_lock_limit;
 
 
3640
3641	lock_limit = rlimit(RLIMIT_MEMLOCK);
3642	lock_limit >>= PAGE_SHIFT;
3643	locked = vma->vm_mm->pinned_vm + extra;
3644
3645	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3646		!capable(CAP_IPC_LOCK)) {
3647		ret = -EPERM;
3648		goto unlock;
3649	}
3650
3651	WARN_ON(event->rb);
3652
3653	if (vma->vm_flags & VM_WRITE)
3654		flags |= RING_BUFFER_WRITABLE;
3655
3656	rb = rb_alloc(nr_pages, 
3657		event->attr.watermark ? event->attr.wakeup_watermark : 0,
3658		event->cpu, flags);
3659
3660	if (!rb) {
3661		ret = -ENOMEM;
3662		goto unlock;
3663	}
3664	rcu_assign_pointer(event->rb, rb);
3665
3666	atomic_long_add(user_extra, &user->locked_vm);
3667	event->mmap_locked = extra;
3668	event->mmap_user = get_current_user();
3669	vma->vm_mm->pinned_vm += event->mmap_locked;
3670
3671	perf_event_update_userpage(event);
 
 
 
 
 
 
 
 
 
 
 
 
 
3672
3673unlock:
3674	if (!ret)
 
 
 
3675		atomic_inc(&event->mmap_count);
 
 
 
 
3676	mutex_unlock(&event->mmap_mutex);
3677
3678	vma->vm_flags |= VM_RESERVED;
 
 
 
 
3679	vma->vm_ops = &perf_mmap_vmops;
3680
 
 
 
3681	return ret;
3682}
3683
3684static int perf_fasync(int fd, struct file *filp, int on)
3685{
3686	struct inode *inode = filp->f_path.dentry->d_inode;
3687	struct perf_event *event = filp->private_data;
3688	int retval;
3689
3690	mutex_lock(&inode->i_mutex);
3691	retval = fasync_helper(fd, filp, on, &event->fasync);
3692	mutex_unlock(&inode->i_mutex);
3693
3694	if (retval < 0)
3695		return retval;
3696
3697	return 0;
3698}
3699
3700static const struct file_operations perf_fops = {
3701	.llseek			= no_llseek,
3702	.release		= perf_release,
3703	.read			= perf_read,
3704	.poll			= perf_poll,
3705	.unlocked_ioctl		= perf_ioctl,
3706	.compat_ioctl		= perf_ioctl,
3707	.mmap			= perf_mmap,
3708	.fasync			= perf_fasync,
3709};
3710
3711/*
3712 * Perf event wakeup
3713 *
3714 * If there's data, ensure we set the poll() state and publish everything
3715 * to user-space before waking everybody up.
3716 */
3717
 
 
 
 
 
 
 
 
3718void perf_event_wakeup(struct perf_event *event)
3719{
3720	ring_buffer_wakeup(event);
3721
3722	if (event->pending_kill) {
3723		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3724		event->pending_kill = 0;
3725	}
3726}
3727
3728static void perf_pending_event(struct irq_work *entry)
3729{
3730	struct perf_event *event = container_of(entry,
3731			struct perf_event, pending);
 
 
3732
3733	if (event->pending_disable) {
3734		event->pending_disable = 0;
3735		__perf_event_disable(event);
 
3736	}
3737
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3738	if (event->pending_wakeup) {
3739		event->pending_wakeup = 0;
3740		perf_event_wakeup(event);
3741	}
 
 
 
3742}
3743
3744/*
3745 * We assume there is only KVM supporting the callbacks.
3746 * Later on, we might change it to a list if there is
3747 * another virtualization implementation supporting the callbacks.
3748 */
3749struct perf_guest_info_callbacks *perf_guest_cbs;
3750
3751int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3752{
3753	perf_guest_cbs = cbs;
3754	return 0;
3755}
3756EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3757
3758int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3759{
3760	perf_guest_cbs = NULL;
3761	return 0;
3762}
3763EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3764
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3765static void __perf_event_header__init_id(struct perf_event_header *header,
3766					 struct perf_sample_data *data,
3767					 struct perf_event *event)
3768{
3769	u64 sample_type = event->attr.sample_type;
3770
3771	data->type = sample_type;
3772	header->size += event->id_header_size;
3773
3774	if (sample_type & PERF_SAMPLE_TID) {
3775		/* namespace issues */
3776		data->tid_entry.pid = perf_event_pid(event, current);
3777		data->tid_entry.tid = perf_event_tid(event, current);
3778	}
3779
3780	if (sample_type & PERF_SAMPLE_TIME)
3781		data->time = perf_clock();
3782
3783	if (sample_type & PERF_SAMPLE_ID)
3784		data->id = primary_event_id(event);
3785
3786	if (sample_type & PERF_SAMPLE_STREAM_ID)
3787		data->stream_id = event->id;
3788
3789	if (sample_type & PERF_SAMPLE_CPU) {
3790		data->cpu_entry.cpu	 = raw_smp_processor_id();
3791		data->cpu_entry.reserved = 0;
3792	}
3793}
3794
3795void perf_event_header__init_id(struct perf_event_header *header,
3796				struct perf_sample_data *data,
3797				struct perf_event *event)
3798{
3799	if (event->attr.sample_id_all)
3800		__perf_event_header__init_id(header, data, event);
3801}
3802
3803static void __perf_event__output_id_sample(struct perf_output_handle *handle,
3804					   struct perf_sample_data *data)
3805{
3806	u64 sample_type = data->type;
3807
3808	if (sample_type & PERF_SAMPLE_TID)
3809		perf_output_put(handle, data->tid_entry);
3810
3811	if (sample_type & PERF_SAMPLE_TIME)
3812		perf_output_put(handle, data->time);
3813
3814	if (sample_type & PERF_SAMPLE_ID)
3815		perf_output_put(handle, data->id);
3816
3817	if (sample_type & PERF_SAMPLE_STREAM_ID)
3818		perf_output_put(handle, data->stream_id);
3819
3820	if (sample_type & PERF_SAMPLE_CPU)
3821		perf_output_put(handle, data->cpu_entry);
 
 
 
3822}
3823
3824void perf_event__output_id_sample(struct perf_event *event,
3825				  struct perf_output_handle *handle,
3826				  struct perf_sample_data *sample)
3827{
3828	if (event->attr.sample_id_all)
3829		__perf_event__output_id_sample(handle, sample);
3830}
3831
3832static void perf_output_read_one(struct perf_output_handle *handle,
3833				 struct perf_event *event,
3834				 u64 enabled, u64 running)
3835{
3836	u64 read_format = event->attr.read_format;
3837	u64 values[4];
3838	int n = 0;
3839
3840	values[n++] = perf_event_count(event);
3841	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3842		values[n++] = enabled +
3843			atomic64_read(&event->child_total_time_enabled);
3844	}
3845	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3846		values[n++] = running +
3847			atomic64_read(&event->child_total_time_running);
3848	}
3849	if (read_format & PERF_FORMAT_ID)
3850		values[n++] = primary_event_id(event);
3851
3852	__output_copy(handle, values, n * sizeof(u64));
3853}
3854
3855/*
3856 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3857 */
3858static void perf_output_read_group(struct perf_output_handle *handle,
3859			    struct perf_event *event,
3860			    u64 enabled, u64 running)
3861{
3862	struct perf_event *leader = event->group_leader, *sub;
3863	u64 read_format = event->attr.read_format;
3864	u64 values[5];
3865	int n = 0;
3866
3867	values[n++] = 1 + leader->nr_siblings;
3868
3869	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3870		values[n++] = enabled;
3871
3872	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3873		values[n++] = running;
3874
3875	if (leader != event)
 
3876		leader->pmu->read(leader);
3877
3878	values[n++] = perf_event_count(leader);
3879	if (read_format & PERF_FORMAT_ID)
3880		values[n++] = primary_event_id(leader);
3881
3882	__output_copy(handle, values, n * sizeof(u64));
3883
3884	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3885		n = 0;
3886
3887		if (sub != event)
 
3888			sub->pmu->read(sub);
3889
3890		values[n++] = perf_event_count(sub);
3891		if (read_format & PERF_FORMAT_ID)
3892			values[n++] = primary_event_id(sub);
3893
3894		__output_copy(handle, values, n * sizeof(u64));
3895	}
3896}
3897
3898#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
3899				 PERF_FORMAT_TOTAL_TIME_RUNNING)
3900
 
 
 
 
 
 
 
3901static void perf_output_read(struct perf_output_handle *handle,
3902			     struct perf_event *event)
3903{
3904	u64 enabled = 0, running = 0, now;
3905	u64 read_format = event->attr.read_format;
3906
3907	/*
3908	 * compute total_time_enabled, total_time_running
3909	 * based on snapshot values taken when the event
3910	 * was last scheduled in.
3911	 *
3912	 * we cannot simply called update_context_time()
3913	 * because of locking issue as we are called in
3914	 * NMI context
3915	 */
3916	if (read_format & PERF_FORMAT_TOTAL_TIMES)
3917		calc_timer_values(event, &now, &enabled, &running);
3918
3919	if (event->attr.read_format & PERF_FORMAT_GROUP)
3920		perf_output_read_group(handle, event, enabled, running);
3921	else
3922		perf_output_read_one(handle, event, enabled, running);
3923}
3924
3925void perf_output_sample(struct perf_output_handle *handle,
3926			struct perf_event_header *header,
3927			struct perf_sample_data *data,
3928			struct perf_event *event)
3929{
3930	u64 sample_type = data->type;
3931
3932	perf_output_put(handle, *header);
3933
 
 
 
3934	if (sample_type & PERF_SAMPLE_IP)
3935		perf_output_put(handle, data->ip);
3936
3937	if (sample_type & PERF_SAMPLE_TID)
3938		perf_output_put(handle, data->tid_entry);
3939
3940	if (sample_type & PERF_SAMPLE_TIME)
3941		perf_output_put(handle, data->time);
3942
3943	if (sample_type & PERF_SAMPLE_ADDR)
3944		perf_output_put(handle, data->addr);
3945
3946	if (sample_type & PERF_SAMPLE_ID)
3947		perf_output_put(handle, data->id);
3948
3949	if (sample_type & PERF_SAMPLE_STREAM_ID)
3950		perf_output_put(handle, data->stream_id);
3951
3952	if (sample_type & PERF_SAMPLE_CPU)
3953		perf_output_put(handle, data->cpu_entry);
3954
3955	if (sample_type & PERF_SAMPLE_PERIOD)
3956		perf_output_put(handle, data->period);
3957
3958	if (sample_type & PERF_SAMPLE_READ)
3959		perf_output_read(handle, event);
3960
3961	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3962		if (data->callchain) {
3963			int size = 1;
3964
3965			if (data->callchain)
3966				size += data->callchain->nr;
3967
3968			size *= sizeof(u64);
3969
3970			__output_copy(handle, data->callchain, size);
3971		} else {
3972			u64 nr = 0;
3973			perf_output_put(handle, nr);
3974		}
3975	}
3976
3977	if (sample_type & PERF_SAMPLE_RAW) {
3978		if (data->raw) {
3979			perf_output_put(handle, data->raw->size);
3980			__output_copy(handle, data->raw->data,
3981					   data->raw->size);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3982		} else {
3983			struct {
3984				u32	size;
3985				u32	data;
3986			} raw = {
3987				.size = sizeof(u32),
3988				.data = 0,
3989			};
3990			perf_output_put(handle, raw);
3991		}
3992	}
3993
3994	if (!event->attr.watermark) {
3995		int wakeup_events = event->attr.wakeup_events;
3996
3997		if (wakeup_events) {
3998			struct ring_buffer *rb = handle->rb;
3999			int events = local_inc_return(&rb->events);
4000
4001			if (events >= wakeup_events) {
4002				local_sub(wakeup_events, &rb->events);
4003				local_inc(&rb->wakeup);
4004			}
4005		}
4006	}
4007
4008	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4009		if (data->br_stack) {
4010			size_t size;
4011
4012			size = data->br_stack->nr
4013			     * sizeof(struct perf_branch_entry);
4014
4015			perf_output_put(handle, data->br_stack->nr);
4016			perf_output_copy(handle, data->br_stack->entries, size);
4017		} else {
4018			/*
4019			 * we always store at least the value of nr
4020			 */
4021			u64 nr = 0;
4022			perf_output_put(handle, nr);
4023		}
4024	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4025}
4026
4027void perf_prepare_sample(struct perf_event_header *header,
4028			 struct perf_sample_data *data,
4029			 struct perf_event *event,
4030			 struct pt_regs *regs)
4031{
4032	u64 sample_type = event->attr.sample_type;
4033
4034	header->type = PERF_RECORD_SAMPLE;
4035	header->size = sizeof(*header) + event->header_size;
4036
4037	header->misc = 0;
4038	header->misc |= perf_misc_flags(regs);
4039
4040	__perf_event_header__init_id(header, data, event);
4041
4042	if (sample_type & PERF_SAMPLE_IP)
4043		data->ip = perf_instruction_pointer(regs);
4044
4045	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4046		int size = 1;
4047
4048		data->callchain = perf_callchain(regs);
 
4049
4050		if (data->callchain)
4051			size += data->callchain->nr;
4052
4053		header->size += size * sizeof(u64);
4054	}
4055
4056	if (sample_type & PERF_SAMPLE_RAW) {
4057		int size = sizeof(u32);
 
4058
4059		if (data->raw)
4060			size += data->raw->size;
4061		else
4062			size += sizeof(u32);
 
 
 
 
 
 
 
 
 
 
 
 
 
4063
4064		WARN_ON_ONCE(size & (sizeof(u64)-1));
4065		header->size += size;
4066	}
4067
4068	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4069		int size = sizeof(u64); /* nr */
4070		if (data->br_stack) {
4071			size += data->br_stack->nr
4072			      * sizeof(struct perf_branch_entry);
4073		}
4074		header->size += size;
4075	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4076}
4077
4078static void perf_event_output(struct perf_event *event,
4079				struct perf_sample_data *data,
4080				struct pt_regs *regs)
 
 
 
 
4081{
4082	struct perf_output_handle handle;
4083	struct perf_event_header header;
 
4084
4085	/* protect the callchain buffers */
4086	rcu_read_lock();
4087
4088	perf_prepare_sample(&header, data, event, regs);
4089
4090	if (perf_output_begin(&handle, event, header.size))
 
4091		goto exit;
4092
4093	perf_output_sample(&handle, &header, data, event);
4094
4095	perf_output_end(&handle);
4096
4097exit:
4098	rcu_read_unlock();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4099}
4100
4101/*
4102 * read event_id
4103 */
4104
4105struct perf_read_event {
4106	struct perf_event_header	header;
4107
4108	u32				pid;
4109	u32				tid;
4110};
4111
4112static void
4113perf_event_read_event(struct perf_event *event,
4114			struct task_struct *task)
4115{
4116	struct perf_output_handle handle;
4117	struct perf_sample_data sample;
4118	struct perf_read_event read_event = {
4119		.header = {
4120			.type = PERF_RECORD_READ,
4121			.misc = 0,
4122			.size = sizeof(read_event) + event->read_size,
4123		},
4124		.pid = perf_event_pid(event, task),
4125		.tid = perf_event_tid(event, task),
4126	};
4127	int ret;
4128
4129	perf_event_header__init_id(&read_event.header, &sample, event);
4130	ret = perf_output_begin(&handle, event, read_event.header.size);
4131	if (ret)
4132		return;
4133
4134	perf_output_put(&handle, read_event);
4135	perf_output_read(&handle, event);
4136	perf_event__output_id_sample(event, &handle, &sample);
4137
4138	perf_output_end(&handle);
4139}
4140
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4141/*
4142 * task tracking -- fork/exit
4143 *
4144 * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
4145 */
4146
4147struct perf_task_event {
4148	struct task_struct		*task;
4149	struct perf_event_context	*task_ctx;
4150
4151	struct {
4152		struct perf_event_header	header;
4153
4154		u32				pid;
4155		u32				ppid;
4156		u32				tid;
4157		u32				ptid;
4158		u64				time;
4159	} event_id;
4160};
4161
 
 
 
 
 
 
 
4162static void perf_event_task_output(struct perf_event *event,
4163				     struct perf_task_event *task_event)
4164{
 
4165	struct perf_output_handle handle;
4166	struct perf_sample_data	sample;
4167	struct task_struct *task = task_event->task;
4168	int ret, size = task_event->event_id.header.size;
4169
 
 
 
4170	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
4171
4172	ret = perf_output_begin(&handle, event,
4173				task_event->event_id.header.size);
4174	if (ret)
4175		goto out;
4176
4177	task_event->event_id.pid = perf_event_pid(event, task);
4178	task_event->event_id.ppid = perf_event_pid(event, current);
4179
4180	task_event->event_id.tid = perf_event_tid(event, task);
4181	task_event->event_id.ptid = perf_event_tid(event, current);
4182
 
 
4183	perf_output_put(&handle, task_event->event_id);
4184
4185	perf_event__output_id_sample(event, &handle, &sample);
4186
4187	perf_output_end(&handle);
4188out:
4189	task_event->event_id.header.size = size;
4190}
4191
4192static int perf_event_task_match(struct perf_event *event)
4193{
4194	if (event->state < PERF_EVENT_STATE_INACTIVE)
4195		return 0;
4196
4197	if (!event_filter_match(event))
4198		return 0;
4199
4200	if (event->attr.comm || event->attr.mmap ||
4201	    event->attr.mmap_data || event->attr.task)
4202		return 1;
4203
4204	return 0;
4205}
4206
4207static void perf_event_task_ctx(struct perf_event_context *ctx,
4208				  struct perf_task_event *task_event)
4209{
4210	struct perf_event *event;
4211
4212	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4213		if (perf_event_task_match(event))
4214			perf_event_task_output(event, task_event);
4215	}
4216}
4217
4218static void perf_event_task_event(struct perf_task_event *task_event)
4219{
4220	struct perf_cpu_context *cpuctx;
4221	struct perf_event_context *ctx;
4222	struct pmu *pmu;
4223	int ctxn;
4224
4225	rcu_read_lock();
4226	list_for_each_entry_rcu(pmu, &pmus, entry) {
4227		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4228		if (cpuctx->active_pmu != pmu)
4229			goto next;
4230		perf_event_task_ctx(&cpuctx->ctx, task_event);
4231
4232		ctx = task_event->task_ctx;
4233		if (!ctx) {
4234			ctxn = pmu->task_ctx_nr;
4235			if (ctxn < 0)
4236				goto next;
4237			ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4238		}
4239		if (ctx)
4240			perf_event_task_ctx(ctx, task_event);
4241next:
4242		put_cpu_ptr(pmu->pmu_cpu_context);
4243	}
4244	rcu_read_unlock();
4245}
4246
4247static void perf_event_task(struct task_struct *task,
4248			      struct perf_event_context *task_ctx,
4249			      int new)
4250{
4251	struct perf_task_event task_event;
4252
4253	if (!atomic_read(&nr_comm_events) &&
4254	    !atomic_read(&nr_mmap_events) &&
4255	    !atomic_read(&nr_task_events))
4256		return;
4257
4258	task_event = (struct perf_task_event){
4259		.task	  = task,
4260		.task_ctx = task_ctx,
4261		.event_id    = {
4262			.header = {
4263				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4264				.misc = 0,
4265				.size = sizeof(task_event.event_id),
4266			},
4267			/* .pid  */
4268			/* .ppid */
4269			/* .tid  */
4270			/* .ptid */
4271			.time = perf_clock(),
4272		},
4273	};
4274
4275	perf_event_task_event(&task_event);
 
 
4276}
4277
4278void perf_event_fork(struct task_struct *task)
4279{
4280	perf_event_task(task, NULL, 1);
 
4281}
4282
4283/*
4284 * comm tracking
4285 */
4286
4287struct perf_comm_event {
4288	struct task_struct	*task;
4289	char			*comm;
4290	int			comm_size;
4291
4292	struct {
4293		struct perf_event_header	header;
4294
4295		u32				pid;
4296		u32				tid;
4297	} event_id;
4298};
4299
 
 
 
 
 
4300static void perf_event_comm_output(struct perf_event *event,
4301				     struct perf_comm_event *comm_event)
4302{
 
4303	struct perf_output_handle handle;
4304	struct perf_sample_data sample;
4305	int size = comm_event->event_id.header.size;
4306	int ret;
4307
 
 
 
4308	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4309	ret = perf_output_begin(&handle, event,
4310				comm_event->event_id.header.size);
4311
4312	if (ret)
4313		goto out;
4314
4315	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4316	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4317
4318	perf_output_put(&handle, comm_event->event_id);
4319	__output_copy(&handle, comm_event->comm,
4320				   comm_event->comm_size);
4321
4322	perf_event__output_id_sample(event, &handle, &sample);
4323
4324	perf_output_end(&handle);
4325out:
4326	comm_event->event_id.header.size = size;
4327}
4328
4329static int perf_event_comm_match(struct perf_event *event)
4330{
4331	if (event->state < PERF_EVENT_STATE_INACTIVE)
4332		return 0;
4333
4334	if (!event_filter_match(event))
4335		return 0;
4336
4337	if (event->attr.comm)
4338		return 1;
4339
4340	return 0;
4341}
4342
4343static void perf_event_comm_ctx(struct perf_event_context *ctx,
4344				  struct perf_comm_event *comm_event)
4345{
4346	struct perf_event *event;
4347
4348	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4349		if (perf_event_comm_match(event))
4350			perf_event_comm_output(event, comm_event);
4351	}
4352}
4353
4354static void perf_event_comm_event(struct perf_comm_event *comm_event)
4355{
4356	struct perf_cpu_context *cpuctx;
4357	struct perf_event_context *ctx;
4358	char comm[TASK_COMM_LEN];
4359	unsigned int size;
4360	struct pmu *pmu;
4361	int ctxn;
4362
4363	memset(comm, 0, sizeof(comm));
4364	strlcpy(comm, comm_event->task->comm, sizeof(comm));
4365	size = ALIGN(strlen(comm)+1, sizeof(u64));
4366
4367	comm_event->comm = comm;
4368	comm_event->comm_size = size;
4369
4370	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4371	rcu_read_lock();
4372	list_for_each_entry_rcu(pmu, &pmus, entry) {
4373		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4374		if (cpuctx->active_pmu != pmu)
4375			goto next;
4376		perf_event_comm_ctx(&cpuctx->ctx, comm_event);
4377
4378		ctxn = pmu->task_ctx_nr;
4379		if (ctxn < 0)
4380			goto next;
4381
4382		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4383		if (ctx)
4384			perf_event_comm_ctx(ctx, comm_event);
4385next:
4386		put_cpu_ptr(pmu->pmu_cpu_context);
4387	}
4388	rcu_read_unlock();
4389}
4390
4391void perf_event_comm(struct task_struct *task)
4392{
4393	struct perf_comm_event comm_event;
4394	struct perf_event_context *ctx;
4395	int ctxn;
4396
4397	for_each_task_context_nr(ctxn) {
4398		ctx = task->perf_event_ctxp[ctxn];
4399		if (!ctx)
4400			continue;
4401
4402		perf_event_enable_on_exec(ctx);
4403	}
4404
4405	if (!atomic_read(&nr_comm_events))
4406		return;
4407
4408	comm_event = (struct perf_comm_event){
4409		.task	= task,
4410		/* .comm      */
4411		/* .comm_size */
4412		.event_id  = {
4413			.header = {
4414				.type = PERF_RECORD_COMM,
4415				.misc = 0,
4416				/* .size */
4417			},
4418			/* .pid */
4419			/* .tid */
4420		},
4421	};
4422
4423	perf_event_comm_event(&comm_event);
4424}
4425
4426/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4427 * mmap tracking
4428 */
4429
4430struct perf_mmap_event {
4431	struct vm_area_struct	*vma;
4432
4433	const char		*file_name;
4434	int			file_size;
 
 
 
 
4435
4436	struct {
4437		struct perf_event_header	header;
4438
4439		u32				pid;
4440		u32				tid;
4441		u64				start;
4442		u64				len;
4443		u64				pgoff;
4444	} event_id;
4445};
4446
 
 
 
 
 
 
 
 
 
 
 
4447static void perf_event_mmap_output(struct perf_event *event,
4448				     struct perf_mmap_event *mmap_event)
4449{
 
4450	struct perf_output_handle handle;
4451	struct perf_sample_data sample;
4452	int size = mmap_event->event_id.header.size;
 
4453	int ret;
4454
 
 
 
 
 
 
 
 
 
 
 
 
 
4455	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4456	ret = perf_output_begin(&handle, event,
4457				mmap_event->event_id.header.size);
4458	if (ret)
4459		goto out;
4460
4461	mmap_event->event_id.pid = perf_event_pid(event, current);
4462	mmap_event->event_id.tid = perf_event_tid(event, current);
4463
4464	perf_output_put(&handle, mmap_event->event_id);
 
 
 
 
 
 
 
 
 
 
4465	__output_copy(&handle, mmap_event->file_name,
4466				   mmap_event->file_size);
4467
4468	perf_event__output_id_sample(event, &handle, &sample);
4469
4470	perf_output_end(&handle);
4471out:
4472	mmap_event->event_id.header.size = size;
4473}
4474
4475static int perf_event_mmap_match(struct perf_event *event,
4476				   struct perf_mmap_event *mmap_event,
4477				   int executable)
4478{
4479	if (event->state < PERF_EVENT_STATE_INACTIVE)
4480		return 0;
4481
4482	if (!event_filter_match(event))
4483		return 0;
4484
4485	if ((!executable && event->attr.mmap_data) ||
4486	    (executable && event->attr.mmap))
4487		return 1;
4488
4489	return 0;
4490}
4491
4492static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4493				  struct perf_mmap_event *mmap_event,
4494				  int executable)
4495{
4496	struct perf_event *event;
4497
4498	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4499		if (perf_event_mmap_match(event, mmap_event, executable))
4500			perf_event_mmap_output(event, mmap_event);
4501	}
4502}
4503
4504static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4505{
4506	struct perf_cpu_context *cpuctx;
4507	struct perf_event_context *ctx;
4508	struct vm_area_struct *vma = mmap_event->vma;
4509	struct file *file = vma->vm_file;
 
 
 
4510	unsigned int size;
4511	char tmp[16];
4512	char *buf = NULL;
4513	const char *name;
4514	struct pmu *pmu;
4515	int ctxn;
 
 
 
 
 
 
 
 
 
 
4516
4517	memset(tmp, 0, sizeof(tmp));
 
 
 
 
 
 
 
4518
4519	if (file) {
 
 
 
 
 
 
 
 
4520		/*
4521		 * d_path works from the end of the rb backwards, so we
4522		 * need to add enough zero bytes after the string to handle
4523		 * the 64bit alignment we do later.
4524		 */
4525		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4526		if (!buf) {
4527			name = strncpy(tmp, "//enomem", sizeof(tmp));
4528			goto got_name;
4529		}
4530		name = d_path(&file->f_path, buf, PATH_MAX);
4531		if (IS_ERR(name)) {
4532			name = strncpy(tmp, "//toolong", sizeof(tmp));
4533			goto got_name;
4534		}
 
 
 
 
 
 
 
 
4535	} else {
4536		if (arch_vma_name(mmap_event->vma)) {
4537			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4538				       sizeof(tmp));
4539			goto got_name;
4540		}
4541
4542		if (!vma->vm_mm) {
4543			name = strncpy(tmp, "[vdso]", sizeof(tmp));
4544			goto got_name;
4545		} else if (vma->vm_start <= vma->vm_mm->start_brk &&
 
4546				vma->vm_end >= vma->vm_mm->brk) {
4547			name = strncpy(tmp, "[heap]", sizeof(tmp));
4548			goto got_name;
4549		} else if (vma->vm_start <= vma->vm_mm->start_stack &&
 
4550				vma->vm_end >= vma->vm_mm->start_stack) {
4551			name = strncpy(tmp, "[stack]", sizeof(tmp));
4552			goto got_name;
4553		}
4554
4555		name = strncpy(tmp, "//anon", sizeof(tmp));
4556		goto got_name;
4557	}
4558
 
 
 
4559got_name:
4560	size = ALIGN(strlen(name)+1, sizeof(u64));
 
 
 
 
 
 
 
4561
4562	mmap_event->file_name = name;
4563	mmap_event->file_size = size;
 
 
 
 
 
 
 
 
 
4564
4565	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4566
4567	rcu_read_lock();
4568	list_for_each_entry_rcu(pmu, &pmus, entry) {
4569		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4570		if (cpuctx->active_pmu != pmu)
4571			goto next;
4572		perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4573					vma->vm_flags & VM_EXEC);
4574
4575		ctxn = pmu->task_ctx_nr;
4576		if (ctxn < 0)
4577			goto next;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4578
 
 
4579		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4580		if (ctx) {
4581			perf_event_mmap_ctx(ctx, mmap_event,
4582					vma->vm_flags & VM_EXEC);
4583		}
4584next:
4585		put_cpu_ptr(pmu->pmu_cpu_context);
4586	}
4587	rcu_read_unlock();
4588
4589	kfree(buf);
4590}
4591
4592void perf_event_mmap(struct vm_area_struct *vma)
4593{
4594	struct perf_mmap_event mmap_event;
4595
4596	if (!atomic_read(&nr_mmap_events))
4597		return;
4598
4599	mmap_event = (struct perf_mmap_event){
4600		.vma	= vma,
4601		/* .file_name */
4602		/* .file_size */
4603		.event_id  = {
4604			.header = {
4605				.type = PERF_RECORD_MMAP,
4606				.misc = PERF_RECORD_MISC_USER,
4607				/* .size */
4608			},
4609			/* .pid */
4610			/* .tid */
4611			.start  = vma->vm_start,
4612			.len    = vma->vm_end - vma->vm_start,
4613			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4614		},
 
 
 
 
 
 
4615	};
4616
 
4617	perf_event_mmap_event(&mmap_event);
4618}
4619
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4620/*
4621 * IRQ throttle logging
4622 */
4623
4624static void perf_log_throttle(struct perf_event *event, int enable)
4625{
4626	struct perf_output_handle handle;
4627	struct perf_sample_data sample;
4628	int ret;
4629
4630	struct {
4631		struct perf_event_header	header;
4632		u64				time;
4633		u64				id;
4634		u64				stream_id;
4635	} throttle_event = {
4636		.header = {
4637			.type = PERF_RECORD_THROTTLE,
4638			.misc = 0,
4639			.size = sizeof(throttle_event),
4640		},
4641		.time		= perf_clock(),
4642		.id		= primary_event_id(event),
4643		.stream_id	= event->id,
4644	};
4645
4646	if (enable)
4647		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4648
4649	perf_event_header__init_id(&throttle_event.header, &sample, event);
4650
4651	ret = perf_output_begin(&handle, event,
4652				throttle_event.header.size);
4653	if (ret)
4654		return;
4655
4656	perf_output_put(&handle, throttle_event);
4657	perf_event__output_id_sample(event, &handle, &sample);
4658	perf_output_end(&handle);
4659}
4660
4661/*
4662 * Generic event overflow handling, sampling.
4663 */
4664
4665static int __perf_event_overflow(struct perf_event *event,
4666				   int throttle, struct perf_sample_data *data,
4667				   struct pt_regs *regs)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4668{
4669	int events = atomic_read(&event->event_limit);
4670	struct hw_perf_event *hwc = &event->hw;
4671	u64 seq;
4672	int ret = 0;
4673
4674	/*
4675	 * Non-sampling counters might still use the PMI to fold short
4676	 * hardware counters, ignore those.
4677	 */
4678	if (unlikely(!is_sampling_event(event)))
4679		return 0;
4680
4681	seq = __this_cpu_read(perf_throttled_seq);
4682	if (seq != hwc->interrupts_seq) {
4683		hwc->interrupts_seq = seq;
4684		hwc->interrupts = 1;
4685	} else {
4686		hwc->interrupts++;
4687		if (unlikely(throttle
4688			     && hwc->interrupts >= max_samples_per_tick)) {
4689			__this_cpu_inc(perf_throttled_count);
 
4690			hwc->interrupts = MAX_INTERRUPTS;
4691			perf_log_throttle(event, 0);
4692			ret = 1;
4693		}
4694	}
4695
4696	if (event->attr.freq) {
4697		u64 now = perf_clock();
4698		s64 delta = now - hwc->freq_time_stamp;
4699
4700		hwc->freq_time_stamp = now;
4701
4702		if (delta > 0 && delta < 2*TICK_NSEC)
4703			perf_adjust_period(event, delta, hwc->last_period, true);
4704	}
4705
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4706	/*
4707	 * XXX event_limit might not quite work as expected on inherited
4708	 * events
4709	 */
4710
4711	event->pending_kill = POLL_IN;
4712	if (events && atomic_dec_and_test(&event->event_limit)) {
4713		ret = 1;
4714		event->pending_kill = POLL_HUP;
4715		event->pending_disable = 1;
4716		irq_work_queue(&event->pending);
4717	}
4718
4719	if (event->overflow_handler)
4720		event->overflow_handler(event, data, regs);
4721	else
4722		perf_event_output(event, data, regs);
4723
4724	if (event->fasync && event->pending_kill) {
4725		event->pending_wakeup = 1;
4726		irq_work_queue(&event->pending);
4727	}
4728
4729	return ret;
4730}
4731
4732int perf_event_overflow(struct perf_event *event,
4733			  struct perf_sample_data *data,
4734			  struct pt_regs *regs)
4735{
4736	return __perf_event_overflow(event, 1, data, regs);
4737}
4738
4739/*
4740 * Generic software event infrastructure
4741 */
4742
4743struct swevent_htable {
4744	struct swevent_hlist		*swevent_hlist;
4745	struct mutex			hlist_mutex;
4746	int				hlist_refcount;
4747
4748	/* Recursion avoidance in each contexts */
4749	int				recursion[PERF_NR_CONTEXTS];
4750};
4751
4752static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
4753
4754/*
4755 * We directly increment event->count and keep a second value in
4756 * event->hw.period_left to count intervals. This period event
4757 * is kept in the range [-sample_period, 0] so that we can use the
4758 * sign as trigger.
4759 */
4760
4761static u64 perf_swevent_set_period(struct perf_event *event)
4762{
4763	struct hw_perf_event *hwc = &event->hw;
4764	u64 period = hwc->last_period;
4765	u64 nr, offset;
4766	s64 old, val;
4767
4768	hwc->last_period = hwc->sample_period;
4769
4770again:
4771	old = val = local64_read(&hwc->period_left);
4772	if (val < 0)
4773		return 0;
4774
4775	nr = div64_u64(period + val, period);
4776	offset = nr * period;
4777	val -= offset;
4778	if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4779		goto again;
4780
4781	return nr;
4782}
4783
4784static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4785				    struct perf_sample_data *data,
4786				    struct pt_regs *regs)
4787{
4788	struct hw_perf_event *hwc = &event->hw;
4789	int throttle = 0;
4790
4791	if (!overflow)
4792		overflow = perf_swevent_set_period(event);
4793
4794	if (hwc->interrupts == MAX_INTERRUPTS)
4795		return;
4796
4797	for (; overflow; overflow--) {
4798		if (__perf_event_overflow(event, throttle,
4799					    data, regs)) {
4800			/*
4801			 * We inhibit the overflow from happening when
4802			 * hwc->interrupts == MAX_INTERRUPTS.
4803			 */
4804			break;
4805		}
4806		throttle = 1;
4807	}
4808}
4809
4810static void perf_swevent_event(struct perf_event *event, u64 nr,
4811			       struct perf_sample_data *data,
4812			       struct pt_regs *regs)
4813{
4814	struct hw_perf_event *hwc = &event->hw;
4815
4816	local64_add(nr, &event->count);
4817
4818	if (!regs)
4819		return;
4820
4821	if (!is_sampling_event(event))
4822		return;
4823
4824	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
4825		data->period = nr;
4826		return perf_swevent_overflow(event, 1, data, regs);
4827	} else
4828		data->period = event->hw.last_period;
4829
4830	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4831		return perf_swevent_overflow(event, 1, data, regs);
4832
4833	if (local64_add_negative(nr, &hwc->period_left))
4834		return;
4835
4836	perf_swevent_overflow(event, 0, data, regs);
4837}
4838
4839static int perf_exclude_event(struct perf_event *event,
4840			      struct pt_regs *regs)
4841{
4842	if (event->hw.state & PERF_HES_STOPPED)
4843		return 1;
4844
4845	if (regs) {
4846		if (event->attr.exclude_user && user_mode(regs))
4847			return 1;
4848
4849		if (event->attr.exclude_kernel && !user_mode(regs))
4850			return 1;
4851	}
4852
4853	return 0;
4854}
4855
4856static int perf_swevent_match(struct perf_event *event,
4857				enum perf_type_id type,
4858				u32 event_id,
4859				struct perf_sample_data *data,
4860				struct pt_regs *regs)
4861{
4862	if (event->attr.type != type)
4863		return 0;
4864
4865	if (event->attr.config != event_id)
4866		return 0;
4867
4868	if (perf_exclude_event(event, regs))
4869		return 0;
4870
4871	return 1;
4872}
4873
4874static inline u64 swevent_hash(u64 type, u32 event_id)
4875{
4876	u64 val = event_id | (type << 32);
4877
4878	return hash_64(val, SWEVENT_HLIST_BITS);
4879}
4880
4881static inline struct hlist_head *
4882__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4883{
4884	u64 hash = swevent_hash(type, event_id);
4885
4886	return &hlist->heads[hash];
4887}
4888
4889/* For the read side: events when they trigger */
4890static inline struct hlist_head *
4891find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4892{
4893	struct swevent_hlist *hlist;
4894
4895	hlist = rcu_dereference(swhash->swevent_hlist);
4896	if (!hlist)
4897		return NULL;
4898
4899	return __find_swevent_head(hlist, type, event_id);
4900}
4901
4902/* For the event head insertion and removal in the hlist */
4903static inline struct hlist_head *
4904find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4905{
4906	struct swevent_hlist *hlist;
4907	u32 event_id = event->attr.config;
4908	u64 type = event->attr.type;
4909
4910	/*
4911	 * Event scheduling is always serialized against hlist allocation
4912	 * and release. Which makes the protected version suitable here.
4913	 * The context lock guarantees that.
4914	 */
4915	hlist = rcu_dereference_protected(swhash->swevent_hlist,
4916					  lockdep_is_held(&event->ctx->lock));
4917	if (!hlist)
4918		return NULL;
4919
4920	return __find_swevent_head(hlist, type, event_id);
4921}
4922
4923static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4924				    u64 nr,
4925				    struct perf_sample_data *data,
4926				    struct pt_regs *regs)
4927{
4928	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4929	struct perf_event *event;
4930	struct hlist_node *node;
4931	struct hlist_head *head;
4932
4933	rcu_read_lock();
4934	head = find_swevent_head_rcu(swhash, type, event_id);
4935	if (!head)
4936		goto end;
4937
4938	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4939		if (perf_swevent_match(event, type, event_id, data, regs))
4940			perf_swevent_event(event, nr, data, regs);
4941	}
4942end:
4943	rcu_read_unlock();
4944}
4945
 
 
4946int perf_swevent_get_recursion_context(void)
4947{
4948	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4949
4950	return get_recursion_context(swhash->recursion);
4951}
4952EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
4953
4954inline void perf_swevent_put_recursion_context(int rctx)
4955{
4956	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4957
4958	put_recursion_context(swhash->recursion, rctx);
4959}
4960
4961void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
4962{
4963	struct perf_sample_data data;
4964	int rctx;
4965
4966	preempt_disable_notrace();
4967	rctx = perf_swevent_get_recursion_context();
4968	if (rctx < 0)
4969		return;
4970
4971	perf_sample_data_init(&data, addr, 0);
4972
4973	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
 
 
 
 
 
 
 
 
 
 
 
 
4974
4975	perf_swevent_put_recursion_context(rctx);
 
4976	preempt_enable_notrace();
4977}
4978
4979static void perf_swevent_read(struct perf_event *event)
4980{
4981}
4982
4983static int perf_swevent_add(struct perf_event *event, int flags)
4984{
4985	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4986	struct hw_perf_event *hwc = &event->hw;
4987	struct hlist_head *head;
4988
4989	if (is_sampling_event(event)) {
4990		hwc->last_period = hwc->sample_period;
4991		perf_swevent_set_period(event);
4992	}
4993
4994	hwc->state = !(flags & PERF_EF_START);
4995
4996	head = find_swevent_head(swhash, event);
4997	if (WARN_ON_ONCE(!head))
4998		return -EINVAL;
4999
5000	hlist_add_head_rcu(&event->hlist_entry, head);
 
5001
5002	return 0;
5003}
5004
5005static void perf_swevent_del(struct perf_event *event, int flags)
5006{
5007	hlist_del_rcu(&event->hlist_entry);
5008}
5009
5010static void perf_swevent_start(struct perf_event *event, int flags)
5011{
5012	event->hw.state = 0;
5013}
5014
5015static void perf_swevent_stop(struct perf_event *event, int flags)
5016{
5017	event->hw.state = PERF_HES_STOPPED;
5018}
5019
5020/* Deref the hlist from the update side */
5021static inline struct swevent_hlist *
5022swevent_hlist_deref(struct swevent_htable *swhash)
5023{
5024	return rcu_dereference_protected(swhash->swevent_hlist,
5025					 lockdep_is_held(&swhash->hlist_mutex));
5026}
5027
5028static void swevent_hlist_release(struct swevent_htable *swhash)
5029{
5030	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5031
5032	if (!hlist)
5033		return;
5034
5035	rcu_assign_pointer(swhash->swevent_hlist, NULL);
5036	kfree_rcu(hlist, rcu_head);
5037}
5038
5039static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5040{
5041	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5042
5043	mutex_lock(&swhash->hlist_mutex);
5044
5045	if (!--swhash->hlist_refcount)
5046		swevent_hlist_release(swhash);
5047
5048	mutex_unlock(&swhash->hlist_mutex);
5049}
5050
5051static void swevent_hlist_put(struct perf_event *event)
5052{
5053	int cpu;
5054
5055	if (event->cpu != -1) {
5056		swevent_hlist_put_cpu(event, event->cpu);
5057		return;
5058	}
5059
5060	for_each_possible_cpu(cpu)
5061		swevent_hlist_put_cpu(event, cpu);
5062}
5063
5064static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
5065{
5066	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5067	int err = 0;
5068
5069	mutex_lock(&swhash->hlist_mutex);
5070
5071	if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5072		struct swevent_hlist *hlist;
5073
5074		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5075		if (!hlist) {
5076			err = -ENOMEM;
5077			goto exit;
5078		}
5079		rcu_assign_pointer(swhash->swevent_hlist, hlist);
5080	}
5081	swhash->hlist_refcount++;
5082exit:
5083	mutex_unlock(&swhash->hlist_mutex);
5084
5085	return err;
5086}
5087
5088static int swevent_hlist_get(struct perf_event *event)
5089{
5090	int err;
5091	int cpu, failed_cpu;
5092
5093	if (event->cpu != -1)
5094		return swevent_hlist_get_cpu(event, event->cpu);
5095
5096	get_online_cpus();
5097	for_each_possible_cpu(cpu) {
5098		err = swevent_hlist_get_cpu(event, cpu);
5099		if (err) {
5100			failed_cpu = cpu;
5101			goto fail;
5102		}
5103	}
5104	put_online_cpus();
5105
5106	return 0;
5107fail:
5108	for_each_possible_cpu(cpu) {
5109		if (cpu == failed_cpu)
5110			break;
5111		swevent_hlist_put_cpu(event, cpu);
5112	}
5113
5114	put_online_cpus();
5115	return err;
5116}
5117
5118struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
5119
5120static void sw_perf_event_destroy(struct perf_event *event)
5121{
5122	u64 event_id = event->attr.config;
5123
5124	WARN_ON(event->parent);
5125
5126	static_key_slow_dec(&perf_swevent_enabled[event_id]);
5127	swevent_hlist_put(event);
5128}
5129
5130static int perf_swevent_init(struct perf_event *event)
5131{
5132	int event_id = event->attr.config;
5133
5134	if (event->attr.type != PERF_TYPE_SOFTWARE)
5135		return -ENOENT;
5136
5137	/*
5138	 * no branch sampling for software events
5139	 */
5140	if (has_branch_stack(event))
5141		return -EOPNOTSUPP;
5142
5143	switch (event_id) {
5144	case PERF_COUNT_SW_CPU_CLOCK:
5145	case PERF_COUNT_SW_TASK_CLOCK:
5146		return -ENOENT;
5147
5148	default:
5149		break;
5150	}
5151
5152	if (event_id >= PERF_COUNT_SW_MAX)
5153		return -ENOENT;
5154
5155	if (!event->parent) {
5156		int err;
5157
5158		err = swevent_hlist_get(event);
5159		if (err)
5160			return err;
5161
5162		static_key_slow_inc(&perf_swevent_enabled[event_id]);
5163		event->destroy = sw_perf_event_destroy;
5164	}
5165
5166	return 0;
5167}
5168
5169static int perf_swevent_event_idx(struct perf_event *event)
5170{
5171	return 0;
5172}
5173
5174static struct pmu perf_swevent = {
5175	.task_ctx_nr	= perf_sw_context,
5176
 
 
5177	.event_init	= perf_swevent_init,
5178	.add		= perf_swevent_add,
5179	.del		= perf_swevent_del,
5180	.start		= perf_swevent_start,
5181	.stop		= perf_swevent_stop,
5182	.read		= perf_swevent_read,
5183
5184	.event_idx	= perf_swevent_event_idx,
5185};
5186
5187#ifdef CONFIG_EVENT_TRACING
5188
5189static int perf_tp_filter_match(struct perf_event *event,
5190				struct perf_sample_data *data)
5191{
5192	void *record = data->raw->data;
 
 
 
 
5193
5194	if (likely(!event->filter) || filter_match_preds(event->filter, record))
5195		return 1;
5196	return 0;
5197}
5198
5199static int perf_tp_event_match(struct perf_event *event,
5200				struct perf_sample_data *data,
5201				struct pt_regs *regs)
5202{
5203	if (event->hw.state & PERF_HES_STOPPED)
5204		return 0;
5205	/*
5206	 * All tracepoints are from kernel-space.
5207	 */
5208	if (event->attr.exclude_kernel)
5209		return 0;
5210
5211	if (!perf_tp_filter_match(event, data))
5212		return 0;
5213
5214	return 1;
5215}
5216
5217void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5218		   struct pt_regs *regs, struct hlist_head *head, int rctx)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5219{
5220	struct perf_sample_data data;
5221	struct perf_event *event;
5222	struct hlist_node *node;
5223
5224	struct perf_raw_record raw = {
5225		.size = entry_size,
5226		.data = record,
 
 
5227	};
5228
5229	perf_sample_data_init(&data, addr, 0);
5230	data.raw = &raw;
5231
5232	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
 
 
5233		if (perf_tp_event_match(event, &data, regs))
5234			perf_swevent_event(event, count, &data, regs);
5235	}
5236
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5237	perf_swevent_put_recursion_context(rctx);
5238}
5239EXPORT_SYMBOL_GPL(perf_tp_event);
5240
5241static void tp_perf_event_destroy(struct perf_event *event)
5242{
5243	perf_trace_destroy(event);
5244}
5245
5246static int perf_tp_event_init(struct perf_event *event)
5247{
5248	int err;
5249
5250	if (event->attr.type != PERF_TYPE_TRACEPOINT)
5251		return -ENOENT;
5252
5253	/*
5254	 * no branch sampling for tracepoint events
5255	 */
5256	if (has_branch_stack(event))
5257		return -EOPNOTSUPP;
5258
5259	err = perf_trace_init(event);
5260	if (err)
5261		return err;
5262
5263	event->destroy = tp_perf_event_destroy;
5264
5265	return 0;
5266}
5267
5268static struct pmu perf_tracepoint = {
5269	.task_ctx_nr	= perf_sw_context,
5270
5271	.event_init	= perf_tp_event_init,
5272	.add		= perf_trace_add,
5273	.del		= perf_trace_del,
5274	.start		= perf_swevent_start,
5275	.stop		= perf_swevent_stop,
5276	.read		= perf_swevent_read,
 
5277
5278	.event_idx	= perf_swevent_event_idx,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5279};
5280
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5281static inline void perf_tp_register(void)
5282{
5283	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
 
 
 
 
 
 
5284}
5285
5286static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5287{
5288	char *filter_str;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5289	int ret;
5290
5291	if (event->attr.type != PERF_TYPE_TRACEPOINT)
 
 
 
 
 
 
 
5292		return -EINVAL;
5293
5294	filter_str = strndup_user(arg, PAGE_SIZE);
5295	if (IS_ERR(filter_str))
5296		return PTR_ERR(filter_str);
 
 
 
 
 
 
 
 
5297
5298	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
 
 
 
 
 
5299
5300	kfree(filter_str);
 
 
 
 
 
 
 
 
 
 
 
5301	return ret;
5302}
5303
5304static void perf_event_free_filter(struct perf_event *event)
5305{
5306	ftrace_profile_free_filter(event);
 
 
 
 
5307}
5308
5309#else
5310
5311static inline void perf_tp_register(void)
5312{
5313}
5314
5315static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5316{
5317	return -ENOENT;
5318}
5319
5320static void perf_event_free_filter(struct perf_event *event)
5321{
 
5322}
5323
 
 
 
5324#endif /* CONFIG_EVENT_TRACING */
5325
5326#ifdef CONFIG_HAVE_HW_BREAKPOINT
5327void perf_bp_event(struct perf_event *bp, void *data)
5328{
5329	struct perf_sample_data sample;
5330	struct pt_regs *regs = data;
5331
5332	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
5333
5334	if (!bp->hw.state && !perf_exclude_event(bp, regs))
5335		perf_swevent_event(bp, 1, &sample, regs);
5336}
5337#endif
5338
5339/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5340 * hrtimer based swevent callback
5341 */
5342
5343static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5344{
5345	enum hrtimer_restart ret = HRTIMER_RESTART;
5346	struct perf_sample_data data;
5347	struct pt_regs *regs;
5348	struct perf_event *event;
5349	u64 period;
5350
5351	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
5352
5353	if (event->state != PERF_EVENT_STATE_ACTIVE)
5354		return HRTIMER_NORESTART;
5355
5356	event->pmu->read(event);
5357
5358	perf_sample_data_init(&data, 0, event->hw.last_period);
5359	regs = get_irq_regs();
5360
5361	if (regs && !perf_exclude_event(event, regs)) {
5362		if (!(event->attr.exclude_idle && is_idle_task(current)))
5363			if (__perf_event_overflow(event, 1, &data, regs))
5364				ret = HRTIMER_NORESTART;
5365	}
5366
5367	period = max_t(u64, 10000, event->hw.sample_period);
5368	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5369
5370	return ret;
5371}
5372
5373static void perf_swevent_start_hrtimer(struct perf_event *event)
5374{
5375	struct hw_perf_event *hwc = &event->hw;
5376	s64 period;
5377
5378	if (!is_sampling_event(event))
5379		return;
5380
5381	period = local64_read(&hwc->period_left);
5382	if (period) {
5383		if (period < 0)
5384			period = 10000;
5385
5386		local64_set(&hwc->period_left, 0);
5387	} else {
5388		period = max_t(u64, 10000, hwc->sample_period);
5389	}
5390	__hrtimer_start_range_ns(&hwc->hrtimer,
5391				ns_to_ktime(period), 0,
5392				HRTIMER_MODE_REL_PINNED, 0);
5393}
5394
5395static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5396{
5397	struct hw_perf_event *hwc = &event->hw;
5398
5399	if (is_sampling_event(event)) {
5400		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
5401		local64_set(&hwc->period_left, ktime_to_ns(remaining));
5402
5403		hrtimer_cancel(&hwc->hrtimer);
5404	}
5405}
5406
5407static void perf_swevent_init_hrtimer(struct perf_event *event)
5408{
5409	struct hw_perf_event *hwc = &event->hw;
5410
5411	if (!is_sampling_event(event))
5412		return;
5413
5414	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5415	hwc->hrtimer.function = perf_swevent_hrtimer;
5416
5417	/*
5418	 * Since hrtimers have a fixed rate, we can do a static freq->period
5419	 * mapping and avoid the whole period adjust feedback stuff.
5420	 */
5421	if (event->attr.freq) {
5422		long freq = event->attr.sample_freq;
5423
5424		event->attr.sample_period = NSEC_PER_SEC / freq;
5425		hwc->sample_period = event->attr.sample_period;
5426		local64_set(&hwc->period_left, hwc->sample_period);
 
5427		event->attr.freq = 0;
5428	}
5429}
5430
5431/*
5432 * Software event: cpu wall time clock
5433 */
5434
5435static void cpu_clock_event_update(struct perf_event *event)
5436{
5437	s64 prev;
5438	u64 now;
5439
5440	now = local_clock();
5441	prev = local64_xchg(&event->hw.prev_count, now);
5442	local64_add(now - prev, &event->count);
5443}
5444
5445static void cpu_clock_event_start(struct perf_event *event, int flags)
5446{
5447	local64_set(&event->hw.prev_count, local_clock());
5448	perf_swevent_start_hrtimer(event);
5449}
5450
5451static void cpu_clock_event_stop(struct perf_event *event, int flags)
5452{
5453	perf_swevent_cancel_hrtimer(event);
5454	cpu_clock_event_update(event);
5455}
5456
5457static int cpu_clock_event_add(struct perf_event *event, int flags)
5458{
5459	if (flags & PERF_EF_START)
5460		cpu_clock_event_start(event, flags);
 
5461
5462	return 0;
5463}
5464
5465static void cpu_clock_event_del(struct perf_event *event, int flags)
5466{
5467	cpu_clock_event_stop(event, flags);
5468}
5469
5470static void cpu_clock_event_read(struct perf_event *event)
5471{
5472	cpu_clock_event_update(event);
5473}
5474
5475static int cpu_clock_event_init(struct perf_event *event)
5476{
5477	if (event->attr.type != PERF_TYPE_SOFTWARE)
5478		return -ENOENT;
5479
5480	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5481		return -ENOENT;
5482
5483	/*
5484	 * no branch sampling for software events
5485	 */
5486	if (has_branch_stack(event))
5487		return -EOPNOTSUPP;
5488
5489	perf_swevent_init_hrtimer(event);
5490
5491	return 0;
5492}
5493
5494static struct pmu perf_cpu_clock = {
5495	.task_ctx_nr	= perf_sw_context,
5496
 
 
5497	.event_init	= cpu_clock_event_init,
5498	.add		= cpu_clock_event_add,
5499	.del		= cpu_clock_event_del,
5500	.start		= cpu_clock_event_start,
5501	.stop		= cpu_clock_event_stop,
5502	.read		= cpu_clock_event_read,
5503
5504	.event_idx	= perf_swevent_event_idx,
5505};
5506
5507/*
5508 * Software event: task time clock
5509 */
5510
5511static void task_clock_event_update(struct perf_event *event, u64 now)
5512{
5513	u64 prev;
5514	s64 delta;
5515
5516	prev = local64_xchg(&event->hw.prev_count, now);
5517	delta = now - prev;
5518	local64_add(delta, &event->count);
5519}
5520
5521static void task_clock_event_start(struct perf_event *event, int flags)
5522{
5523	local64_set(&event->hw.prev_count, event->ctx->time);
5524	perf_swevent_start_hrtimer(event);
5525}
5526
5527static void task_clock_event_stop(struct perf_event *event, int flags)
5528{
5529	perf_swevent_cancel_hrtimer(event);
5530	task_clock_event_update(event, event->ctx->time);
5531}
5532
5533static int task_clock_event_add(struct perf_event *event, int flags)
5534{
5535	if (flags & PERF_EF_START)
5536		task_clock_event_start(event, flags);
 
5537
5538	return 0;
5539}
5540
5541static void task_clock_event_del(struct perf_event *event, int flags)
5542{
5543	task_clock_event_stop(event, PERF_EF_UPDATE);
5544}
5545
5546static void task_clock_event_read(struct perf_event *event)
5547{
5548	u64 now = perf_clock();
5549	u64 delta = now - event->ctx->timestamp;
5550	u64 time = event->ctx->time + delta;
5551
5552	task_clock_event_update(event, time);
5553}
5554
5555static int task_clock_event_init(struct perf_event *event)
5556{
5557	if (event->attr.type != PERF_TYPE_SOFTWARE)
5558		return -ENOENT;
5559
5560	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5561		return -ENOENT;
5562
5563	/*
5564	 * no branch sampling for software events
5565	 */
5566	if (has_branch_stack(event))
5567		return -EOPNOTSUPP;
5568
5569	perf_swevent_init_hrtimer(event);
5570
5571	return 0;
5572}
5573
5574static struct pmu perf_task_clock = {
5575	.task_ctx_nr	= perf_sw_context,
5576
 
 
5577	.event_init	= task_clock_event_init,
5578	.add		= task_clock_event_add,
5579	.del		= task_clock_event_del,
5580	.start		= task_clock_event_start,
5581	.stop		= task_clock_event_stop,
5582	.read		= task_clock_event_read,
5583
5584	.event_idx	= perf_swevent_event_idx,
5585};
5586
5587static void perf_pmu_nop_void(struct pmu *pmu)
5588{
5589}
5590
 
 
 
 
5591static int perf_pmu_nop_int(struct pmu *pmu)
5592{
5593	return 0;
5594}
5595
5596static void perf_pmu_start_txn(struct pmu *pmu)
5597{
 
 
 
 
 
 
 
 
 
 
 
 
5598	perf_pmu_disable(pmu);
5599}
5600
5601static int perf_pmu_commit_txn(struct pmu *pmu)
5602{
 
 
 
 
 
 
 
5603	perf_pmu_enable(pmu);
5604	return 0;
5605}
5606
5607static void perf_pmu_cancel_txn(struct pmu *pmu)
5608{
 
 
 
 
 
 
 
5609	perf_pmu_enable(pmu);
5610}
5611
5612static int perf_event_idx_default(struct perf_event *event)
5613{
5614	return event->hw.idx + 1;
5615}
5616
5617/*
5618 * Ensures all contexts with the same task_ctx_nr have the same
5619 * pmu_cpu_context too.
5620 */
5621static void *find_pmu_context(int ctxn)
5622{
5623	struct pmu *pmu;
5624
5625	if (ctxn < 0)
5626		return NULL;
5627
5628	list_for_each_entry(pmu, &pmus, entry) {
5629		if (pmu->task_ctx_nr == ctxn)
5630			return pmu->pmu_cpu_context;
5631	}
5632
5633	return NULL;
5634}
5635
5636static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5637{
5638	int cpu;
5639
5640	for_each_possible_cpu(cpu) {
5641		struct perf_cpu_context *cpuctx;
5642
5643		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5644
5645		if (cpuctx->active_pmu == old_pmu)
5646			cpuctx->active_pmu = pmu;
5647	}
5648}
5649
5650static void free_pmu_context(struct pmu *pmu)
5651{
5652	struct pmu *i;
5653
5654	mutex_lock(&pmus_lock);
5655	/*
5656	 * Like a real lame refcount.
 
 
5657	 */
5658	list_for_each_entry(i, &pmus, entry) {
5659		if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5660			update_pmu_context(i, pmu);
5661			goto out;
5662		}
5663	}
5664
5665	free_percpu(pmu->pmu_cpu_context);
5666out:
5667	mutex_unlock(&pmus_lock);
5668}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5669static struct idr pmu_idr;
5670
5671static ssize_t
5672type_show(struct device *dev, struct device_attribute *attr, char *page)
5673{
5674	struct pmu *pmu = dev_get_drvdata(dev);
5675
5676	return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
5677}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5678
5679static struct device_attribute pmu_dev_attrs[] = {
5680       __ATTR_RO(type),
5681       __ATTR_NULL,
 
5682};
 
5683
5684static int pmu_bus_running;
5685static struct bus_type pmu_bus = {
5686	.name		= "event_source",
5687	.dev_attrs	= pmu_dev_attrs,
5688};
5689
5690static void pmu_dev_release(struct device *dev)
5691{
5692	kfree(dev);
5693}
5694
5695static int pmu_dev_alloc(struct pmu *pmu)
5696{
5697	int ret = -ENOMEM;
5698
5699	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
5700	if (!pmu->dev)
5701		goto out;
5702
5703	pmu->dev->groups = pmu->attr_groups;
5704	device_initialize(pmu->dev);
5705	ret = dev_set_name(pmu->dev, "%s", pmu->name);
5706	if (ret)
5707		goto free_dev;
5708
5709	dev_set_drvdata(pmu->dev, pmu);
5710	pmu->dev->bus = &pmu_bus;
5711	pmu->dev->release = pmu_dev_release;
5712	ret = device_add(pmu->dev);
5713	if (ret)
5714		goto free_dev;
5715
 
 
 
 
 
 
 
 
 
 
 
 
 
5716out:
5717	return ret;
5718
 
 
 
5719free_dev:
5720	put_device(pmu->dev);
5721	goto out;
5722}
5723
5724static struct lock_class_key cpuctx_mutex;
5725static struct lock_class_key cpuctx_lock;
5726
5727int perf_pmu_register(struct pmu *pmu, char *name, int type)
5728{
5729	int cpu, ret;
5730
5731	mutex_lock(&pmus_lock);
5732	ret = -ENOMEM;
5733	pmu->pmu_disable_count = alloc_percpu(int);
5734	if (!pmu->pmu_disable_count)
5735		goto unlock;
5736
5737	pmu->type = -1;
5738	if (!name)
5739		goto skip_type;
5740	pmu->name = name;
5741
5742	if (type < 0) {
5743		int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
5744		if (!err)
5745			goto free_pdc;
5746
5747		err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
5748		if (err) {
5749			ret = err;
5750			goto free_pdc;
5751		}
5752	}
5753	pmu->type = type;
5754
5755	if (pmu_bus_running) {
5756		ret = pmu_dev_alloc(pmu);
5757		if (ret)
5758			goto free_idr;
5759	}
5760
5761skip_type:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5762	pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
5763	if (pmu->pmu_cpu_context)
5764		goto got_cpu_context;
5765
 
5766	pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
5767	if (!pmu->pmu_cpu_context)
5768		goto free_dev;
5769
5770	for_each_possible_cpu(cpu) {
5771		struct perf_cpu_context *cpuctx;
5772
5773		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5774		__perf_event_init_context(&cpuctx->ctx);
5775		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5776		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
5777		cpuctx->ctx.type = cpu_context;
5778		cpuctx->ctx.pmu = pmu;
5779		cpuctx->jiffies_interval = 1;
5780		INIT_LIST_HEAD(&cpuctx->rotation_list);
5781		cpuctx->active_pmu = pmu;
5782	}
5783
5784got_cpu_context:
5785	if (!pmu->start_txn) {
5786		if (pmu->pmu_enable) {
5787			/*
5788			 * If we have pmu_enable/pmu_disable calls, install
5789			 * transaction stubs that use that to try and batch
5790			 * hardware accesses.
5791			 */
5792			pmu->start_txn  = perf_pmu_start_txn;
5793			pmu->commit_txn = perf_pmu_commit_txn;
5794			pmu->cancel_txn = perf_pmu_cancel_txn;
5795		} else {
5796			pmu->start_txn  = perf_pmu_nop_void;
5797			pmu->commit_txn = perf_pmu_nop_int;
5798			pmu->cancel_txn = perf_pmu_nop_void;
5799		}
5800	}
5801
5802	if (!pmu->pmu_enable) {
5803		pmu->pmu_enable  = perf_pmu_nop_void;
5804		pmu->pmu_disable = perf_pmu_nop_void;
5805	}
5806
 
 
 
5807	if (!pmu->event_idx)
5808		pmu->event_idx = perf_event_idx_default;
5809
5810	list_add_rcu(&pmu->entry, &pmus);
 
5811	ret = 0;
5812unlock:
5813	mutex_unlock(&pmus_lock);
5814
5815	return ret;
5816
5817free_dev:
5818	device_del(pmu->dev);
5819	put_device(pmu->dev);
5820
5821free_idr:
5822	if (pmu->type >= PERF_TYPE_MAX)
5823		idr_remove(&pmu_idr, pmu->type);
5824
5825free_pdc:
5826	free_percpu(pmu->pmu_disable_count);
5827	goto unlock;
5828}
 
5829
5830void perf_pmu_unregister(struct pmu *pmu)
5831{
5832	mutex_lock(&pmus_lock);
5833	list_del_rcu(&pmu->entry);
5834	mutex_unlock(&pmus_lock);
5835
5836	/*
5837	 * We dereference the pmu list under both SRCU and regular RCU, so
5838	 * synchronize against both of those.
5839	 */
5840	synchronize_srcu(&pmus_srcu);
5841	synchronize_rcu();
5842
5843	free_percpu(pmu->pmu_disable_count);
5844	if (pmu->type >= PERF_TYPE_MAX)
5845		idr_remove(&pmu_idr, pmu->type);
5846	device_del(pmu->dev);
5847	put_device(pmu->dev);
 
 
 
 
5848	free_pmu_context(pmu);
 
5849}
 
5850
5851struct pmu *perf_init_event(struct perf_event *event)
5852{
5853	struct pmu *pmu = NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5854	int idx;
5855	int ret;
5856
5857	idx = srcu_read_lock(&pmus_srcu);
5858
 
 
 
 
 
 
 
 
5859	rcu_read_lock();
5860	pmu = idr_find(&pmu_idr, event->attr.type);
5861	rcu_read_unlock();
5862	if (pmu) {
5863		event->pmu = pmu;
5864		ret = pmu->event_init(event);
5865		if (ret)
5866			pmu = ERR_PTR(ret);
5867		goto unlock;
5868	}
5869
5870	list_for_each_entry_rcu(pmu, &pmus, entry) {
5871		event->pmu = pmu;
5872		ret = pmu->event_init(event);
5873		if (!ret)
5874			goto unlock;
5875
5876		if (ret != -ENOENT) {
5877			pmu = ERR_PTR(ret);
5878			goto unlock;
5879		}
5880	}
5881	pmu = ERR_PTR(-ENOENT);
5882unlock:
5883	srcu_read_unlock(&pmus_srcu, idx);
5884
5885	return pmu;
5886}
5887
 
 
 
 
 
 
 
 
 
5888/*
5889 * Allocate and initialize a event structure
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5890 */
5891static struct perf_event *
5892perf_event_alloc(struct perf_event_attr *attr, int cpu,
5893		 struct task_struct *task,
5894		 struct perf_event *group_leader,
5895		 struct perf_event *parent_event,
5896		 perf_overflow_handler_t overflow_handler,
5897		 void *context)
5898{
5899	struct pmu *pmu;
5900	struct perf_event *event;
5901	struct hw_perf_event *hwc;
5902	long err;
5903
5904	if ((unsigned)cpu >= nr_cpu_ids) {
5905		if (!task || cpu != -1)
5906			return ERR_PTR(-EINVAL);
5907	}
5908
5909	event = kzalloc(sizeof(*event), GFP_KERNEL);
5910	if (!event)
5911		return ERR_PTR(-ENOMEM);
5912
5913	/*
5914	 * Single events are their own group leaders, with an
5915	 * empty sibling list:
5916	 */
5917	if (!group_leader)
5918		group_leader = event;
5919
5920	mutex_init(&event->child_mutex);
5921	INIT_LIST_HEAD(&event->child_list);
5922
5923	INIT_LIST_HEAD(&event->group_entry);
5924	INIT_LIST_HEAD(&event->event_entry);
5925	INIT_LIST_HEAD(&event->sibling_list);
 
 
5926	INIT_LIST_HEAD(&event->rb_entry);
 
 
 
 
5927
5928	init_waitqueue_head(&event->waitq);
 
5929	init_irq_work(&event->pending, perf_pending_event);
5930
5931	mutex_init(&event->mmap_mutex);
 
5932
5933	atomic_long_set(&event->refcount, 1);
5934	event->cpu		= cpu;
5935	event->attr		= *attr;
5936	event->group_leader	= group_leader;
5937	event->pmu		= NULL;
5938	event->oncpu		= -1;
5939
5940	event->parent		= parent_event;
5941
5942	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
5943	event->id		= atomic64_inc_return(&perf_event_id);
5944
5945	event->state		= PERF_EVENT_STATE_INACTIVE;
5946
5947	if (task) {
5948		event->attach_state = PERF_ATTACH_TASK;
5949#ifdef CONFIG_HAVE_HW_BREAKPOINT
5950		/*
5951		 * hw_breakpoint is a bit difficult here..
 
 
5952		 */
5953		if (attr->type == PERF_TYPE_BREAKPOINT)
5954			event->hw.bp_target = task;
5955#endif
5956	}
5957
 
 
 
 
5958	if (!overflow_handler && parent_event) {
5959		overflow_handler = parent_event->overflow_handler;
5960		context = parent_event->overflow_handler_context;
 
 
 
 
 
 
 
 
 
 
 
 
 
5961	}
5962
5963	event->overflow_handler	= overflow_handler;
5964	event->overflow_handler_context = context;
 
 
 
 
 
 
 
 
5965
5966	if (attr->disabled)
5967		event->state = PERF_EVENT_STATE_OFF;
5968
5969	pmu = NULL;
5970
5971	hwc = &event->hw;
5972	hwc->sample_period = attr->sample_period;
5973	if (attr->freq && attr->sample_freq)
5974		hwc->sample_period = 1;
5975	hwc->last_period = hwc->sample_period;
5976
5977	local64_set(&hwc->period_left, hwc->sample_period);
5978
5979	/*
5980	 * we currently do not support PERF_FORMAT_GROUP on inherited events
 
5981	 */
5982	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5983		goto done;
 
 
 
 
 
 
 
 
 
5984
5985	pmu = perf_init_event(event);
 
 
 
 
5986
5987done:
5988	err = 0;
5989	if (!pmu)
 
 
5990		err = -EINVAL;
5991	else if (IS_ERR(pmu))
5992		err = PTR_ERR(pmu);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5993
5994	if (err) {
5995		if (event->ns)
5996			put_pid_ns(event->ns);
5997		kfree(event);
5998		return ERR_PTR(err);
5999	}
6000
6001	if (!event->parent) {
6002		if (event->attach_state & PERF_ATTACH_TASK)
6003			static_key_slow_inc(&perf_sched_events.key);
6004		if (event->attr.mmap || event->attr.mmap_data)
6005			atomic_inc(&nr_mmap_events);
6006		if (event->attr.comm)
6007			atomic_inc(&nr_comm_events);
6008		if (event->attr.task)
6009			atomic_inc(&nr_task_events);
6010		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
6011			err = get_callchain_buffers();
6012			if (err) {
6013				free_event(event);
6014				return ERR_PTR(err);
6015			}
6016		}
6017		if (has_branch_stack(event)) {
6018			static_key_slow_inc(&perf_sched_events.key);
6019			if (!(event->attach_state & PERF_ATTACH_TASK))
6020				atomic_inc(&per_cpu(perf_branch_stack_events,
6021						    event->cpu));
6022		}
6023	}
6024
 
 
 
6025	return event;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6026}
6027
6028static int perf_copy_attr(struct perf_event_attr __user *uattr,
6029			  struct perf_event_attr *attr)
6030{
6031	u32 size;
6032	int ret;
6033
6034	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
6035		return -EFAULT;
6036
6037	/*
6038	 * zero the full structure, so that a short copy will be nice.
6039	 */
6040	memset(attr, 0, sizeof(*attr));
6041
6042	ret = get_user(size, &uattr->size);
6043	if (ret)
6044		return ret;
6045
6046	if (size > PAGE_SIZE)	/* silly large */
6047		goto err_size;
6048
6049	if (!size)		/* abi compat */
6050		size = PERF_ATTR_SIZE_VER0;
6051
6052	if (size < PERF_ATTR_SIZE_VER0)
6053		goto err_size;
6054
6055	/*
6056	 * If we're handed a bigger struct than we know of,
6057	 * ensure all the unknown bits are 0 - i.e. new
6058	 * user-space does not rely on any kernel feature
6059	 * extensions we dont know about yet.
6060	 */
6061	if (size > sizeof(*attr)) {
6062		unsigned char __user *addr;
6063		unsigned char __user *end;
6064		unsigned char val;
6065
6066		addr = (void __user *)uattr + sizeof(*attr);
6067		end  = (void __user *)uattr + size;
6068
6069		for (; addr < end; addr++) {
6070			ret = get_user(val, addr);
6071			if (ret)
6072				return ret;
6073			if (val)
6074				goto err_size;
6075		}
6076		size = sizeof(*attr);
6077	}
6078
6079	ret = copy_from_user(attr, uattr, size);
6080	if (ret)
6081		return -EFAULT;
6082
6083	if (attr->__reserved_1)
6084		return -EINVAL;
6085
6086	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
6087		return -EINVAL;
6088
6089	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
6090		return -EINVAL;
6091
6092	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
6093		u64 mask = attr->branch_sample_type;
6094
6095		/* only using defined bits */
6096		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
6097			return -EINVAL;
6098
6099		/* at least one branch bit must be set */
6100		if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
6101			return -EINVAL;
6102
6103		/* kernel level capture: check permissions */
6104		if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
6105		    && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6106			return -EACCES;
6107
6108		/* propagate priv level, when not set for branch */
6109		if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
6110
6111			/* exclude_kernel checked on syscall entry */
6112			if (!attr->exclude_kernel)
6113				mask |= PERF_SAMPLE_BRANCH_KERNEL;
6114
6115			if (!attr->exclude_user)
6116				mask |= PERF_SAMPLE_BRANCH_USER;
6117
6118			if (!attr->exclude_hv)
6119				mask |= PERF_SAMPLE_BRANCH_HV;
6120			/*
6121			 * adjust user setting (for HW filter setup)
6122			 */
6123			attr->branch_sample_type = mask;
6124		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6125	}
 
 
 
 
 
 
6126out:
6127	return ret;
6128
6129err_size:
6130	put_user(sizeof(*attr), &uattr->size);
6131	ret = -E2BIG;
6132	goto out;
6133}
6134
6135static int
6136perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
6137{
6138	struct ring_buffer *rb = NULL, *old_rb = NULL;
6139	int ret = -EINVAL;
6140
6141	if (!output_event)
6142		goto set;
6143
6144	/* don't allow circular references */
6145	if (event == output_event)
6146		goto out;
6147
6148	/*
6149	 * Don't allow cross-cpu buffers
6150	 */
6151	if (output_event->cpu != event->cpu)
6152		goto out;
6153
6154	/*
6155	 * If its not a per-cpu rb, it must be the same task.
6156	 */
6157	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
6158		goto out;
6159
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6160set:
6161	mutex_lock(&event->mmap_mutex);
6162	/* Can't redirect output if we've got an active mmap() */
6163	if (atomic_read(&event->mmap_count))
6164		goto unlock;
6165
6166	if (output_event) {
6167		/* get the rb we want to redirect to */
6168		rb = ring_buffer_get(output_event);
6169		if (!rb)
6170			goto unlock;
6171	}
6172
6173	old_rb = event->rb;
6174	rcu_assign_pointer(event->rb, rb);
6175	if (old_rb)
6176		ring_buffer_detach(event, old_rb);
6177	ret = 0;
6178unlock:
6179	mutex_unlock(&event->mmap_mutex);
6180
6181	if (old_rb)
6182		ring_buffer_put(old_rb);
6183out:
6184	return ret;
6185}
6186
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6187/**
6188 * sys_perf_event_open - open a performance event, associate it to a task/cpu
6189 *
6190 * @attr_uptr:	event_id type attributes for monitoring/sampling
6191 * @pid:		target pid
6192 * @cpu:		target cpu
6193 * @group_fd:		group leader event fd
6194 */
6195SYSCALL_DEFINE5(perf_event_open,
6196		struct perf_event_attr __user *, attr_uptr,
6197		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
6198{
6199	struct perf_event *group_leader = NULL, *output_event = NULL;
6200	struct perf_event *event, *sibling;
6201	struct perf_event_attr attr;
6202	struct perf_event_context *ctx;
6203	struct file *event_file = NULL;
6204	struct file *group_file = NULL;
6205	struct task_struct *task = NULL;
6206	struct pmu *pmu;
6207	int event_fd;
6208	int move_group = 0;
6209	int fput_needed = 0;
6210	int err;
 
 
6211
6212	/* for future expandability... */
6213	if (flags & ~PERF_FLAG_ALL)
6214		return -EINVAL;
6215
6216	err = perf_copy_attr(attr_uptr, &attr);
6217	if (err)
6218		return err;
6219
6220	if (!attr.exclude_kernel) {
6221		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6222			return -EACCES;
6223	}
6224
 
 
 
 
 
6225	if (attr.freq) {
6226		if (attr.sample_freq > sysctl_perf_event_sample_rate)
6227			return -EINVAL;
 
 
 
6228	}
6229
 
 
 
 
 
 
 
 
 
 
 
 
6230	/*
6231	 * In cgroup mode, the pid argument is used to pass the fd
6232	 * opened to the cgroup directory in cgroupfs. The cpu argument
6233	 * designates the cpu on which to monitor threads from that
6234	 * cgroup.
6235	 */
6236	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
6237		return -EINVAL;
6238
6239	event_fd = get_unused_fd_flags(O_RDWR);
 
 
 
6240	if (event_fd < 0)
6241		return event_fd;
6242
6243	if (group_fd != -1) {
6244		group_file = perf_fget_light(group_fd, &fput_needed);
6245		if (IS_ERR(group_file)) {
6246			err = PTR_ERR(group_file);
6247			goto err_fd;
6248		}
6249		group_leader = group_file->private_data;
6250		if (flags & PERF_FLAG_FD_OUTPUT)
6251			output_event = group_leader;
6252		if (flags & PERF_FLAG_FD_NO_GROUP)
6253			group_leader = NULL;
6254	}
6255
6256	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
6257		task = find_lively_task_by_vpid(pid);
6258		if (IS_ERR(task)) {
6259			err = PTR_ERR(task);
6260			goto err_group_fd;
6261		}
6262	}
6263
6264	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
6265				 NULL, NULL);
6266	if (IS_ERR(event)) {
6267		err = PTR_ERR(event);
6268		goto err_task;
6269	}
6270
6271	if (flags & PERF_FLAG_PID_CGROUP) {
6272		err = perf_cgroup_connect(pid, event, &attr, group_leader);
6273		if (err)
6274			goto err_alloc;
 
6275		/*
6276		 * one more event:
6277		 * - that has cgroup constraint on event->cpu
6278		 * - that may need work on context switch
 
 
 
6279		 */
6280		atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
6281		static_key_slow_inc(&perf_sched_events.key);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6282	}
6283
6284	/*
6285	 * Special case software events and allow them to be part of
6286	 * any hardware group.
6287	 */
6288	pmu = event->pmu;
6289
6290	if (group_leader &&
6291	    (is_software_event(event) != is_software_event(group_leader))) {
6292		if (is_software_event(event)) {
 
 
 
 
 
 
 
 
 
6293			/*
6294			 * If event and group_leader are not both a software
6295			 * event, and event is, then group leader is not.
6296			 *
6297			 * Allow the addition of software events to !software
6298			 * groups, this is safe because software events never
6299			 * fail to schedule.
6300			 */
6301			pmu = group_leader->pmu;
6302		} else if (is_software_event(group_leader) &&
6303			   (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
 
6304			/*
6305			 * In case the group is a pure software group, and we
6306			 * try to add a hardware event, move the whole group to
6307			 * the hardware context.
6308			 */
6309			move_group = 1;
6310		}
6311	}
6312
6313	/*
6314	 * Get the target context (task or percpu):
6315	 */
6316	ctx = find_get_context(pmu, task, cpu);
6317	if (IS_ERR(ctx)) {
6318		err = PTR_ERR(ctx);
6319		goto err_alloc;
6320	}
6321
6322	if (task) {
6323		put_task_struct(task);
6324		task = NULL;
6325	}
6326
6327	/*
6328	 * Look up the group leader (we will attach this event to it):
6329	 */
6330	if (group_leader) {
6331		err = -EINVAL;
6332
6333		/*
6334		 * Do not allow a recursive hierarchy (this new sibling
6335		 * becoming part of another group-sibling):
6336		 */
6337		if (group_leader->group_leader != group_leader)
6338			goto err_context;
 
 
 
 
 
6339		/*
6340		 * Do not allow to attach to a group in a different
6341		 * task or CPU context:
 
6342		 */
6343		if (move_group) {
6344			if (group_leader->ctx->type != ctx->type)
6345				goto err_context;
6346		} else {
6347			if (group_leader->ctx != ctx)
6348				goto err_context;
6349		}
 
 
 
 
 
 
 
 
 
 
6350
6351		/*
6352		 * Only a group leader can be exclusive or pinned
6353		 */
6354		if (attr.exclusive || attr.pinned)
6355			goto err_context;
6356	}
6357
6358	if (output_event) {
6359		err = perf_event_set_output(event, output_event);
6360		if (err)
6361			goto err_context;
6362	}
6363
6364	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
 
6365	if (IS_ERR(event_file)) {
6366		err = PTR_ERR(event_file);
 
6367		goto err_context;
6368	}
6369
6370	if (move_group) {
6371		struct perf_event_context *gctx = group_leader->ctx;
6372
6373		mutex_lock(&gctx->mutex);
6374		perf_remove_from_context(group_leader);
6375		list_for_each_entry(sibling, &group_leader->sibling_list,
6376				    group_entry) {
6377			perf_remove_from_context(sibling);
6378			put_ctx(gctx);
6379		}
6380		mutex_unlock(&gctx->mutex);
6381		put_ctx(gctx);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6382	}
6383
6384	WARN_ON_ONCE(ctx->parent_ctx);
6385	mutex_lock(&ctx->mutex);
 
 
 
 
6386
6387	if (move_group) {
6388		perf_install_in_context(ctx, group_leader, cpu);
6389		get_ctx(ctx);
6390		list_for_each_entry(sibling, &group_leader->sibling_list,
6391				    group_entry) {
6392			perf_install_in_context(ctx, sibling, cpu);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6393			get_ctx(ctx);
6394		}
 
 
 
 
 
 
 
 
 
6395	}
6396
6397	perf_install_in_context(ctx, event, cpu);
6398	++ctx->generation;
 
 
 
 
 
 
 
 
 
 
6399	perf_unpin_context(ctx);
 
 
 
6400	mutex_unlock(&ctx->mutex);
6401
6402	event->owner = current;
 
 
 
6403
6404	mutex_lock(&current->perf_event_mutex);
6405	list_add_tail(&event->owner_entry, &current->perf_event_list);
6406	mutex_unlock(&current->perf_event_mutex);
6407
6408	/*
6409	 * Precalculate sample_data sizes
6410	 */
6411	perf_event__header_size(event);
6412	perf_event__id_header_size(event);
6413
6414	/*
6415	 * Drop the reference on the group_event after placing the
6416	 * new event on the sibling_list. This ensures destruction
6417	 * of the group leader will find the pointer to itself in
6418	 * perf_group_detach().
6419	 */
6420	fput_light(group_file, fput_needed);
6421	fd_install(event_fd, event_file);
6422	return event_fd;
6423
 
 
 
 
 
 
6424err_context:
6425	perf_unpin_context(ctx);
6426	put_ctx(ctx);
6427err_alloc:
6428	free_event(event);
 
 
 
 
 
 
 
 
6429err_task:
6430	if (task)
6431		put_task_struct(task);
6432err_group_fd:
6433	fput_light(group_file, fput_needed);
6434err_fd:
6435	put_unused_fd(event_fd);
6436	return err;
6437}
6438
6439/**
6440 * perf_event_create_kernel_counter
6441 *
6442 * @attr: attributes of the counter to create
6443 * @cpu: cpu in which the counter is bound
6444 * @task: task to profile (NULL for percpu)
6445 */
6446struct perf_event *
6447perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
6448				 struct task_struct *task,
6449				 perf_overflow_handler_t overflow_handler,
6450				 void *context)
6451{
6452	struct perf_event_context *ctx;
6453	struct perf_event *event;
6454	int err;
6455
6456	/*
6457	 * Get the target context (task or percpu):
 
6458	 */
 
 
6459
6460	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
6461				 overflow_handler, context);
6462	if (IS_ERR(event)) {
6463		err = PTR_ERR(event);
6464		goto err;
6465	}
6466
6467	ctx = find_get_context(event->pmu, task, cpu);
 
 
 
 
 
 
6468	if (IS_ERR(ctx)) {
6469		err = PTR_ERR(ctx);
6470		goto err_free;
6471	}
6472
6473	WARN_ON_ONCE(ctx->parent_ctx);
6474	mutex_lock(&ctx->mutex);
6475	perf_install_in_context(ctx, event, cpu);
6476	++ctx->generation;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6477	perf_unpin_context(ctx);
6478	mutex_unlock(&ctx->mutex);
6479
6480	return event;
6481
 
 
 
 
6482err_free:
6483	free_event(event);
6484err:
6485	return ERR_PTR(err);
6486}
6487EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6488
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6489static void sync_child_event(struct perf_event *child_event,
6490			       struct task_struct *child)
6491{
6492	struct perf_event *parent_event = child_event->parent;
6493	u64 child_val;
6494
6495	if (child_event->attr.inherit_stat)
6496		perf_event_read_event(child_event, child);
6497
6498	child_val = perf_event_count(child_event);
6499
6500	/*
6501	 * Add back the child's count to the parent's count:
6502	 */
6503	atomic64_add(child_val, &parent_event->child_count);
6504	atomic64_add(child_event->total_time_enabled,
6505		     &parent_event->child_total_time_enabled);
6506	atomic64_add(child_event->total_time_running,
6507		     &parent_event->child_total_time_running);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6508
6509	/*
6510	 * Remove this event from the parent's list
6511	 */
6512	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6513	mutex_lock(&parent_event->child_mutex);
6514	list_del_init(&child_event->child_list);
6515	mutex_unlock(&parent_event->child_mutex);
6516
6517	/*
6518	 * Release the parent event, if this was the last
6519	 * reference to it.
6520	 */
 
 
6521	put_event(parent_event);
6522}
6523
6524static void
6525__perf_event_exit_task(struct perf_event *child_event,
6526			 struct perf_event_context *child_ctx,
6527			 struct task_struct *child)
6528{
6529	if (child_event->parent) {
6530		raw_spin_lock_irq(&child_ctx->lock);
6531		perf_group_detach(child_event);
6532		raw_spin_unlock_irq(&child_ctx->lock);
6533	}
6534
6535	perf_remove_from_context(child_event);
6536
6537	/*
6538	 * It can happen that the parent exits first, and has events
6539	 * that are still around due to the child reference. These
6540	 * events need to be zapped.
6541	 */
6542	if (child_event->parent) {
6543		sync_child_event(child_event, child);
6544		free_event(child_event);
6545	}
6546}
6547
6548static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6549{
6550	struct perf_event *child_event, *tmp;
6551	struct perf_event_context *child_ctx;
6552	unsigned long flags;
 
6553
6554	if (likely(!child->perf_event_ctxp[ctxn])) {
6555		perf_event_task(child, NULL, 0);
6556		return;
6557	}
6558
6559	local_irq_save(flags);
6560	/*
6561	 * We can't reschedule here because interrupts are disabled,
6562	 * and either child is current or it is a task that can't be
6563	 * scheduled, so we are now safe from rescheduling changing
6564	 * our context.
 
 
 
 
6565	 */
6566	child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6567
6568	/*
6569	 * Take the context lock here so that if find_get_context is
6570	 * reading child->perf_event_ctxp, we wait until it has
6571	 * incremented the context's refcount before we do put_ctx below.
6572	 */
6573	raw_spin_lock(&child_ctx->lock);
6574	task_ctx_sched_out(child_ctx);
6575	child->perf_event_ctxp[ctxn] = NULL;
6576	/*
6577	 * If this context is a clone; unclone it so it can't get
6578	 * swapped to another process while we're removing all
6579	 * the events from it.
6580	 */
6581	unclone_ctx(child_ctx);
6582	update_context_time(child_ctx);
6583	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
 
 
 
 
 
 
 
6584
6585	/*
6586	 * Report the task dead after unscheduling the events so that we
6587	 * won't get any samples after PERF_RECORD_EXIT. We can however still
6588	 * get a few PERF_RECORD_READ events.
6589	 */
6590	perf_event_task(child, child_ctx, 0);
6591
6592	/*
6593	 * We can recurse on the same lock type through:
6594	 *
6595	 *   __perf_event_exit_task()
6596	 *     sync_child_event()
6597	 *       put_event()
6598	 *         mutex_lock(&ctx->mutex)
6599	 *
6600	 * But since its the parent context it won't be the same instance.
6601	 */
6602	mutex_lock(&child_ctx->mutex);
6603
6604again:
6605	list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
6606				 group_entry)
6607		__perf_event_exit_task(child_event, child_ctx, child);
6608
6609	list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6610				 group_entry)
6611		__perf_event_exit_task(child_event, child_ctx, child);
6612
6613	/*
6614	 * If the last event was a group event, it will have appended all
6615	 * its siblings to the list, but we obtained 'tmp' before that which
6616	 * will still point to the list head terminating the iteration.
6617	 */
6618	if (!list_empty(&child_ctx->pinned_groups) ||
6619	    !list_empty(&child_ctx->flexible_groups))
6620		goto again;
6621
6622	mutex_unlock(&child_ctx->mutex);
6623
6624	put_ctx(child_ctx);
6625}
6626
6627/*
6628 * When a child task exits, feed back event values to parent events.
 
 
 
6629 */
6630void perf_event_exit_task(struct task_struct *child)
6631{
6632	struct perf_event *event, *tmp;
6633	int ctxn;
6634
6635	mutex_lock(&child->perf_event_mutex);
6636	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
6637				 owner_entry) {
6638		list_del_init(&event->owner_entry);
6639
6640		/*
6641		 * Ensure the list deletion is visible before we clear
6642		 * the owner, closes a race against perf_release() where
6643		 * we need to serialize on the owner->perf_event_mutex.
6644		 */
6645		smp_wmb();
6646		event->owner = NULL;
6647	}
6648	mutex_unlock(&child->perf_event_mutex);
6649
6650	for_each_task_context_nr(ctxn)
6651		perf_event_exit_task_context(child, ctxn);
 
 
 
 
 
 
 
 
6652}
6653
6654static void perf_free_event(struct perf_event *event,
6655			    struct perf_event_context *ctx)
6656{
6657	struct perf_event *parent = event->parent;
6658
6659	if (WARN_ON_ONCE(!parent))
6660		return;
6661
6662	mutex_lock(&parent->child_mutex);
6663	list_del_init(&event->child_list);
6664	mutex_unlock(&parent->child_mutex);
6665
6666	put_event(parent);
6667
 
6668	perf_group_detach(event);
6669	list_del_event(event, ctx);
 
6670	free_event(event);
6671}
6672
6673/*
6674 * free an unexposed, unused context as created by inheritance by
6675 * perf_event_init_task below, used by fork() in case of fail.
 
 
 
6676 */
6677void perf_event_free_task(struct task_struct *task)
6678{
6679	struct perf_event_context *ctx;
6680	struct perf_event *event, *tmp;
6681	int ctxn;
6682
6683	for_each_task_context_nr(ctxn) {
6684		ctx = task->perf_event_ctxp[ctxn];
6685		if (!ctx)
6686			continue;
6687
6688		mutex_lock(&ctx->mutex);
6689again:
6690		list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
6691				group_entry)
6692			perf_free_event(event, ctx);
 
 
 
 
 
 
 
6693
6694		list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
6695				group_entry)
6696			perf_free_event(event, ctx);
6697
6698		if (!list_empty(&ctx->pinned_groups) ||
6699				!list_empty(&ctx->flexible_groups))
6700			goto again;
6701
6702		mutex_unlock(&ctx->mutex);
6703
6704		put_ctx(ctx);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6705	}
6706}
6707
6708void perf_event_delayed_put(struct task_struct *task)
6709{
6710	int ctxn;
6711
6712	for_each_task_context_nr(ctxn)
6713		WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
6714}
6715
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6716/*
6717 * inherit a event from parent task to child task:
 
 
 
 
 
6718 */
6719static struct perf_event *
6720inherit_event(struct perf_event *parent_event,
6721	      struct task_struct *parent,
6722	      struct perf_event_context *parent_ctx,
6723	      struct task_struct *child,
6724	      struct perf_event *group_leader,
6725	      struct perf_event_context *child_ctx)
6726{
 
6727	struct perf_event *child_event;
6728	unsigned long flags;
6729
6730	/*
6731	 * Instead of creating recursive hierarchies of events,
6732	 * we link inherited events back to the original parent,
6733	 * which has a filp for sure, which we use as the reference
6734	 * count:
6735	 */
6736	if (parent_event->parent)
6737		parent_event = parent_event->parent;
6738
6739	child_event = perf_event_alloc(&parent_event->attr,
6740					   parent_event->cpu,
6741					   child,
6742					   group_leader, parent_event,
6743				           NULL, NULL);
6744	if (IS_ERR(child_event))
6745		return child_event;
6746
6747	if (!atomic_long_inc_not_zero(&parent_event->refcount)) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6748		free_event(child_event);
6749		return NULL;
6750	}
6751
6752	get_ctx(child_ctx);
6753
6754	/*
6755	 * Make the child state follow the state of the parent event,
6756	 * not its attr.disabled bit.  We hold the parent's mutex,
6757	 * so we won't race with perf_event_{en, dis}able_family.
6758	 */
6759	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
6760		child_event->state = PERF_EVENT_STATE_INACTIVE;
6761	else
6762		child_event->state = PERF_EVENT_STATE_OFF;
6763
6764	if (parent_event->attr.freq) {
6765		u64 sample_period = parent_event->hw.sample_period;
6766		struct hw_perf_event *hwc = &child_event->hw;
6767
6768		hwc->sample_period = sample_period;
6769		hwc->last_period   = sample_period;
6770
6771		local64_set(&hwc->period_left, sample_period);
6772	}
6773
6774	child_event->ctx = child_ctx;
6775	child_event->overflow_handler = parent_event->overflow_handler;
6776	child_event->overflow_handler_context
6777		= parent_event->overflow_handler_context;
6778
6779	/*
6780	 * Precalculate sample_data sizes
6781	 */
6782	perf_event__header_size(child_event);
6783	perf_event__id_header_size(child_event);
6784
6785	/*
6786	 * Link it up in the child's context:
6787	 */
6788	raw_spin_lock_irqsave(&child_ctx->lock, flags);
6789	add_event_to_ctx(child_event, child_ctx);
6790	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6791
6792	/*
6793	 * Link this into the parent event's child list
6794	 */
6795	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6796	mutex_lock(&parent_event->child_mutex);
6797	list_add_tail(&child_event->child_list, &parent_event->child_list);
6798	mutex_unlock(&parent_event->child_mutex);
6799
6800	return child_event;
6801}
6802
 
 
 
 
 
 
 
 
 
 
6803static int inherit_group(struct perf_event *parent_event,
6804	      struct task_struct *parent,
6805	      struct perf_event_context *parent_ctx,
6806	      struct task_struct *child,
6807	      struct perf_event_context *child_ctx)
6808{
6809	struct perf_event *leader;
6810	struct perf_event *sub;
6811	struct perf_event *child_ctr;
6812
6813	leader = inherit_event(parent_event, parent, parent_ctx,
6814				 child, NULL, child_ctx);
6815	if (IS_ERR(leader))
6816		return PTR_ERR(leader);
6817	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
 
 
 
 
 
6818		child_ctr = inherit_event(sub, parent, parent_ctx,
6819					    child, leader, child_ctx);
6820		if (IS_ERR(child_ctr))
6821			return PTR_ERR(child_ctr);
 
 
 
 
6822	}
6823	return 0;
6824}
6825
 
 
 
 
 
 
 
 
 
 
 
6826static int
6827inherit_task_group(struct perf_event *event, struct task_struct *parent,
6828		   struct perf_event_context *parent_ctx,
6829		   struct task_struct *child, int ctxn,
6830		   int *inherited_all)
6831{
6832	int ret;
6833	struct perf_event_context *child_ctx;
6834
6835	if (!event->attr.inherit) {
6836		*inherited_all = 0;
6837		return 0;
6838	}
6839
6840	child_ctx = child->perf_event_ctxp[ctxn];
6841	if (!child_ctx) {
6842		/*
6843		 * This is executed from the parent task context, so
6844		 * inherit events that have been marked for cloning.
6845		 * First allocate and initialize a context for the
6846		 * child.
6847		 */
6848
6849		child_ctx = alloc_perf_context(event->pmu, child);
6850		if (!child_ctx)
6851			return -ENOMEM;
6852
6853		child->perf_event_ctxp[ctxn] = child_ctx;
6854	}
6855
6856	ret = inherit_group(event, parent, parent_ctx,
6857			    child, child_ctx);
6858
6859	if (ret)
6860		*inherited_all = 0;
6861
6862	return ret;
6863}
6864
6865/*
6866 * Initialize the perf_event context in task_struct
6867 */
6868int perf_event_init_context(struct task_struct *child, int ctxn)
6869{
6870	struct perf_event_context *child_ctx, *parent_ctx;
6871	struct perf_event_context *cloned_ctx;
6872	struct perf_event *event;
6873	struct task_struct *parent = current;
6874	int inherited_all = 1;
6875	unsigned long flags;
6876	int ret = 0;
6877
6878	if (likely(!parent->perf_event_ctxp[ctxn]))
6879		return 0;
6880
6881	/*
6882	 * If the parent's context is a clone, pin it so it won't get
6883	 * swapped under us.
6884	 */
6885	parent_ctx = perf_pin_task_context(parent, ctxn);
 
 
6886
6887	/*
6888	 * No need to check if parent_ctx != NULL here; since we saw
6889	 * it non-NULL earlier, the only reason for it to become NULL
6890	 * is if we exit, and since we're currently in the middle of
6891	 * a fork we can't be exiting at the same time.
6892	 */
6893
6894	/*
6895	 * Lock the parent list. No need to lock the child - not PID
6896	 * hashed yet and not running, so nobody can access it.
6897	 */
6898	mutex_lock(&parent_ctx->mutex);
6899
6900	/*
6901	 * We dont have to disable NMIs - we are only looking at
6902	 * the list, not manipulating it:
6903	 */
6904	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
6905		ret = inherit_task_group(event, parent, parent_ctx,
6906					 child, ctxn, &inherited_all);
6907		if (ret)
6908			break;
6909	}
6910
6911	/*
6912	 * We can't hold ctx->lock when iterating the ->flexible_group list due
6913	 * to allocations, but we need to prevent rotation because
6914	 * rotate_ctx() will change the list from interrupt context.
6915	 */
6916	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6917	parent_ctx->rotate_disable = 1;
6918	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6919
6920	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
6921		ret = inherit_task_group(event, parent, parent_ctx,
6922					 child, ctxn, &inherited_all);
6923		if (ret)
6924			break;
6925	}
6926
6927	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6928	parent_ctx->rotate_disable = 0;
6929
6930	child_ctx = child->perf_event_ctxp[ctxn];
6931
6932	if (child_ctx && inherited_all) {
6933		/*
6934		 * Mark the child context as a clone of the parent
6935		 * context, or of whatever the parent is a clone of.
6936		 *
6937		 * Note that if the parent is a clone, the holding of
6938		 * parent_ctx->lock avoids it from being uncloned.
6939		 */
6940		cloned_ctx = parent_ctx->parent_ctx;
6941		if (cloned_ctx) {
6942			child_ctx->parent_ctx = cloned_ctx;
6943			child_ctx->parent_gen = parent_ctx->parent_gen;
6944		} else {
6945			child_ctx->parent_ctx = parent_ctx;
6946			child_ctx->parent_gen = parent_ctx->generation;
6947		}
6948		get_ctx(child_ctx->parent_ctx);
6949	}
6950
6951	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
 
6952	mutex_unlock(&parent_ctx->mutex);
6953
6954	perf_unpin_context(parent_ctx);
6955	put_ctx(parent_ctx);
6956
6957	return ret;
6958}
6959
6960/*
6961 * Initialize the perf_event context in task_struct
6962 */
6963int perf_event_init_task(struct task_struct *child)
6964{
6965	int ctxn, ret;
6966
6967	memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
6968	mutex_init(&child->perf_event_mutex);
6969	INIT_LIST_HEAD(&child->perf_event_list);
6970
6971	for_each_task_context_nr(ctxn) {
6972		ret = perf_event_init_context(child, ctxn);
6973		if (ret)
 
6974			return ret;
 
6975	}
6976
6977	return 0;
6978}
6979
6980static void __init perf_event_init_all_cpus(void)
6981{
6982	struct swevent_htable *swhash;
6983	int cpu;
6984
 
 
6985	for_each_possible_cpu(cpu) {
6986		swhash = &per_cpu(swevent_htable, cpu);
6987		mutex_init(&swhash->hlist_mutex);
6988		INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
 
 
 
 
 
 
 
 
6989	}
6990}
6991
6992static void __cpuinit perf_event_init_cpu(int cpu)
6993{
6994	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6995
6996	mutex_lock(&swhash->hlist_mutex);
6997	if (swhash->hlist_refcount > 0) {
6998		struct swevent_hlist *hlist;
6999
7000		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
7001		WARN_ON(!hlist);
7002		rcu_assign_pointer(swhash->swevent_hlist, hlist);
7003	}
7004	mutex_unlock(&swhash->hlist_mutex);
7005}
7006
7007#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
7008static void perf_pmu_rotate_stop(struct pmu *pmu)
7009{
7010	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
7011
7012	WARN_ON(!irqs_disabled());
7013
7014	list_del_init(&cpuctx->rotation_list);
7015}
7016
7017static void __perf_event_exit_context(void *__info)
7018{
7019	struct perf_event_context *ctx = __info;
7020	struct perf_event *event, *tmp;
7021
7022	perf_pmu_rotate_stop(ctx->pmu);
7023
7024	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
7025		__perf_remove_from_context(event);
7026	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
7027		__perf_remove_from_context(event);
 
7028}
7029
7030static void perf_event_exit_cpu_context(int cpu)
7031{
 
7032	struct perf_event_context *ctx;
7033	struct pmu *pmu;
7034	int idx;
7035
7036	idx = srcu_read_lock(&pmus_srcu);
7037	list_for_each_entry_rcu(pmu, &pmus, entry) {
7038		ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
 
7039
7040		mutex_lock(&ctx->mutex);
7041		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
 
7042		mutex_unlock(&ctx->mutex);
7043	}
7044	srcu_read_unlock(&pmus_srcu, idx);
 
7045}
 
7046
7047static void perf_event_exit_cpu(int cpu)
 
 
 
 
7048{
7049	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
 
 
7050
7051	mutex_lock(&swhash->hlist_mutex);
7052	swevent_hlist_release(swhash);
7053	mutex_unlock(&swhash->hlist_mutex);
 
 
 
 
7054
 
 
 
 
 
 
 
 
 
 
 
7055	perf_event_exit_cpu_context(cpu);
 
7056}
7057#else
7058static inline void perf_event_exit_cpu(int cpu) { }
7059#endif
7060
7061static int
7062perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
7063{
7064	int cpu;
7065
7066	for_each_online_cpu(cpu)
7067		perf_event_exit_cpu(cpu);
7068
7069	return NOTIFY_OK;
7070}
7071
7072/*
7073 * Run the perf reboot notifier at the very last possible moment so that
7074 * the generic watchdog code runs as long as possible.
7075 */
7076static struct notifier_block perf_reboot_notifier = {
7077	.notifier_call = perf_reboot,
7078	.priority = INT_MIN,
7079};
7080
7081static int __cpuinit
7082perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
7083{
7084	unsigned int cpu = (long)hcpu;
7085
7086	switch (action & ~CPU_TASKS_FROZEN) {
7087
7088	case CPU_UP_PREPARE:
7089	case CPU_DOWN_FAILED:
7090		perf_event_init_cpu(cpu);
7091		break;
7092
7093	case CPU_UP_CANCELED:
7094	case CPU_DOWN_PREPARE:
7095		perf_event_exit_cpu(cpu);
7096		break;
7097
7098	default:
7099		break;
7100	}
7101
7102	return NOTIFY_OK;
7103}
7104
7105void __init perf_event_init(void)
7106{
7107	int ret;
7108
7109	idr_init(&pmu_idr);
7110
7111	perf_event_init_all_cpus();
7112	init_srcu_struct(&pmus_srcu);
7113	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
7114	perf_pmu_register(&perf_cpu_clock, NULL, -1);
7115	perf_pmu_register(&perf_task_clock, NULL, -1);
7116	perf_tp_register();
7117	perf_cpu_notifier(perf_cpu_notify);
7118	register_reboot_notifier(&perf_reboot_notifier);
7119
7120	ret = init_hw_breakpoint();
7121	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
7122
7123	/* do not patch jump label more than once per second */
7124	jump_label_rate_limit(&perf_sched_events, HZ);
7125
7126	/*
7127	 * Build time assertion that we keep the data_head at the intended
7128	 * location.  IOW, validation we got the __reserved[] size right.
7129	 */
7130	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
7131		     != 1024);
7132}
7133
 
 
 
 
 
 
 
 
 
 
 
 
 
7134static int __init perf_event_sysfs_init(void)
7135{
7136	struct pmu *pmu;
7137	int ret;
7138
7139	mutex_lock(&pmus_lock);
7140
7141	ret = bus_register(&pmu_bus);
7142	if (ret)
7143		goto unlock;
7144
7145	list_for_each_entry(pmu, &pmus, entry) {
7146		if (!pmu->name || pmu->type < 0)
7147			continue;
7148
7149		ret = pmu_dev_alloc(pmu);
7150		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
7151	}
7152	pmu_bus_running = 1;
7153	ret = 0;
7154
7155unlock:
7156	mutex_unlock(&pmus_lock);
7157
7158	return ret;
7159}
7160device_initcall(perf_event_sysfs_init);
7161
7162#ifdef CONFIG_CGROUP_PERF
7163static struct cgroup_subsys_state *perf_cgroup_create(struct cgroup *cont)
 
7164{
7165	struct perf_cgroup *jc;
7166
7167	jc = kzalloc(sizeof(*jc), GFP_KERNEL);
7168	if (!jc)
7169		return ERR_PTR(-ENOMEM);
7170
7171	jc->info = alloc_percpu(struct perf_cgroup_info);
7172	if (!jc->info) {
7173		kfree(jc);
7174		return ERR_PTR(-ENOMEM);
7175	}
7176
7177	return &jc->css;
7178}
7179
7180static void perf_cgroup_destroy(struct cgroup *cont)
7181{
7182	struct perf_cgroup *jc;
7183	jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
7184			  struct perf_cgroup, css);
7185	free_percpu(jc->info);
7186	kfree(jc);
7187}
7188
7189static int __perf_cgroup_move(void *info)
7190{
7191	struct task_struct *task = info;
 
7192	perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
 
7193	return 0;
7194}
7195
7196static void perf_cgroup_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
7197{
7198	struct task_struct *task;
 
7199
7200	cgroup_taskset_for_each(task, cgrp, tset)
7201		task_function_call(task, __perf_cgroup_move, task);
7202}
7203
7204static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
7205			     struct task_struct *task)
7206{
 
7207	/*
7208	 * cgroup_exit() is called in the copy_process() failure path.
7209	 * Ignore this case since the task hasn't ran yet, this avoids
7210	 * trying to poke a half freed task state from generic code.
7211	 */
7212	if (!(task->flags & PF_EXITING))
7213		return;
7214
7215	task_function_call(task, __perf_cgroup_move, task);
7216}
7217
7218struct cgroup_subsys perf_subsys = {
7219	.name		= "perf_event",
7220	.subsys_id	= perf_subsys_id,
7221	.create		= perf_cgroup_create,
7222	.destroy	= perf_cgroup_destroy,
7223	.exit		= perf_cgroup_exit,
7224	.attach		= perf_cgroup_attach,
7225};
7226#endif /* CONFIG_CGROUP_PERF */