Loading...
1/*
2 * Performance events core code:
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8 *
9 * For licensing details see kernel-base/COPYING
10 */
11
12#include <linux/fs.h>
13#include <linux/mm.h>
14#include <linux/cpu.h>
15#include <linux/smp.h>
16#include <linux/idr.h>
17#include <linux/file.h>
18#include <linux/poll.h>
19#include <linux/slab.h>
20#include <linux/hash.h>
21#include <linux/sysfs.h>
22#include <linux/dcache.h>
23#include <linux/percpu.h>
24#include <linux/ptrace.h>
25#include <linux/reboot.h>
26#include <linux/vmstat.h>
27#include <linux/device.h>
28#include <linux/export.h>
29#include <linux/vmalloc.h>
30#include <linux/hardirq.h>
31#include <linux/rculist.h>
32#include <linux/uaccess.h>
33#include <linux/syscalls.h>
34#include <linux/anon_inodes.h>
35#include <linux/kernel_stat.h>
36#include <linux/perf_event.h>
37#include <linux/ftrace_event.h>
38#include <linux/hw_breakpoint.h>
39
40#include "internal.h"
41
42#include <asm/irq_regs.h>
43
44struct remote_function_call {
45 struct task_struct *p;
46 int (*func)(void *info);
47 void *info;
48 int ret;
49};
50
51static void remote_function(void *data)
52{
53 struct remote_function_call *tfc = data;
54 struct task_struct *p = tfc->p;
55
56 if (p) {
57 tfc->ret = -EAGAIN;
58 if (task_cpu(p) != smp_processor_id() || !task_curr(p))
59 return;
60 }
61
62 tfc->ret = tfc->func(tfc->info);
63}
64
65/**
66 * task_function_call - call a function on the cpu on which a task runs
67 * @p: the task to evaluate
68 * @func: the function to be called
69 * @info: the function call argument
70 *
71 * Calls the function @func when the task is currently running. This might
72 * be on the current CPU, which just calls the function directly
73 *
74 * returns: @func return value, or
75 * -ESRCH - when the process isn't running
76 * -EAGAIN - when the process moved away
77 */
78static int
79task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
80{
81 struct remote_function_call data = {
82 .p = p,
83 .func = func,
84 .info = info,
85 .ret = -ESRCH, /* No such (running) process */
86 };
87
88 if (task_curr(p))
89 smp_call_function_single(task_cpu(p), remote_function, &data, 1);
90
91 return data.ret;
92}
93
94/**
95 * cpu_function_call - call a function on the cpu
96 * @func: the function to be called
97 * @info: the function call argument
98 *
99 * Calls the function @func on the remote cpu.
100 *
101 * returns: @func return value or -ENXIO when the cpu is offline
102 */
103static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
104{
105 struct remote_function_call data = {
106 .p = NULL,
107 .func = func,
108 .info = info,
109 .ret = -ENXIO, /* No such CPU */
110 };
111
112 smp_call_function_single(cpu, remote_function, &data, 1);
113
114 return data.ret;
115}
116
117#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
118 PERF_FLAG_FD_OUTPUT |\
119 PERF_FLAG_PID_CGROUP)
120
121/*
122 * branch priv levels that need permission checks
123 */
124#define PERF_SAMPLE_BRANCH_PERM_PLM \
125 (PERF_SAMPLE_BRANCH_KERNEL |\
126 PERF_SAMPLE_BRANCH_HV)
127
128enum event_type_t {
129 EVENT_FLEXIBLE = 0x1,
130 EVENT_PINNED = 0x2,
131 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
132};
133
134/*
135 * perf_sched_events : >0 events exist
136 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
137 */
138struct static_key_deferred perf_sched_events __read_mostly;
139static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
140static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
141
142static atomic_t nr_mmap_events __read_mostly;
143static atomic_t nr_comm_events __read_mostly;
144static atomic_t nr_task_events __read_mostly;
145
146static LIST_HEAD(pmus);
147static DEFINE_MUTEX(pmus_lock);
148static struct srcu_struct pmus_srcu;
149
150/*
151 * perf event paranoia level:
152 * -1 - not paranoid at all
153 * 0 - disallow raw tracepoint access for unpriv
154 * 1 - disallow cpu events for unpriv
155 * 2 - disallow kernel profiling for unpriv
156 */
157int sysctl_perf_event_paranoid __read_mostly = 1;
158
159/* Minimum for 512 kiB + 1 user control page */
160int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
161
162/*
163 * max perf event sample rate
164 */
165#define DEFAULT_MAX_SAMPLE_RATE 100000
166int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
167static int max_samples_per_tick __read_mostly =
168 DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
169
170int perf_proc_update_handler(struct ctl_table *table, int write,
171 void __user *buffer, size_t *lenp,
172 loff_t *ppos)
173{
174 int ret = proc_dointvec(table, write, buffer, lenp, ppos);
175
176 if (ret || !write)
177 return ret;
178
179 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
180
181 return 0;
182}
183
184static atomic64_t perf_event_id;
185
186static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
187 enum event_type_t event_type);
188
189static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
190 enum event_type_t event_type,
191 struct task_struct *task);
192
193static void update_context_time(struct perf_event_context *ctx);
194static u64 perf_event_time(struct perf_event *event);
195
196static void ring_buffer_attach(struct perf_event *event,
197 struct ring_buffer *rb);
198
199void __weak perf_event_print_debug(void) { }
200
201extern __weak const char *perf_pmu_name(void)
202{
203 return "pmu";
204}
205
206static inline u64 perf_clock(void)
207{
208 return local_clock();
209}
210
211static inline struct perf_cpu_context *
212__get_cpu_context(struct perf_event_context *ctx)
213{
214 return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
215}
216
217static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
218 struct perf_event_context *ctx)
219{
220 raw_spin_lock(&cpuctx->ctx.lock);
221 if (ctx)
222 raw_spin_lock(&ctx->lock);
223}
224
225static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
226 struct perf_event_context *ctx)
227{
228 if (ctx)
229 raw_spin_unlock(&ctx->lock);
230 raw_spin_unlock(&cpuctx->ctx.lock);
231}
232
233#ifdef CONFIG_CGROUP_PERF
234
235/*
236 * Must ensure cgroup is pinned (css_get) before calling
237 * this function. In other words, we cannot call this function
238 * if there is no cgroup event for the current CPU context.
239 */
240static inline struct perf_cgroup *
241perf_cgroup_from_task(struct task_struct *task)
242{
243 return container_of(task_subsys_state(task, perf_subsys_id),
244 struct perf_cgroup, css);
245}
246
247static inline bool
248perf_cgroup_match(struct perf_event *event)
249{
250 struct perf_event_context *ctx = event->ctx;
251 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
252
253 return !event->cgrp || event->cgrp == cpuctx->cgrp;
254}
255
256static inline bool perf_tryget_cgroup(struct perf_event *event)
257{
258 return css_tryget(&event->cgrp->css);
259}
260
261static inline void perf_put_cgroup(struct perf_event *event)
262{
263 css_put(&event->cgrp->css);
264}
265
266static inline void perf_detach_cgroup(struct perf_event *event)
267{
268 perf_put_cgroup(event);
269 event->cgrp = NULL;
270}
271
272static inline int is_cgroup_event(struct perf_event *event)
273{
274 return event->cgrp != NULL;
275}
276
277static inline u64 perf_cgroup_event_time(struct perf_event *event)
278{
279 struct perf_cgroup_info *t;
280
281 t = per_cpu_ptr(event->cgrp->info, event->cpu);
282 return t->time;
283}
284
285static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
286{
287 struct perf_cgroup_info *info;
288 u64 now;
289
290 now = perf_clock();
291
292 info = this_cpu_ptr(cgrp->info);
293
294 info->time += now - info->timestamp;
295 info->timestamp = now;
296}
297
298static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
299{
300 struct perf_cgroup *cgrp_out = cpuctx->cgrp;
301 if (cgrp_out)
302 __update_cgrp_time(cgrp_out);
303}
304
305static inline void update_cgrp_time_from_event(struct perf_event *event)
306{
307 struct perf_cgroup *cgrp;
308
309 /*
310 * ensure we access cgroup data only when needed and
311 * when we know the cgroup is pinned (css_get)
312 */
313 if (!is_cgroup_event(event))
314 return;
315
316 cgrp = perf_cgroup_from_task(current);
317 /*
318 * Do not update time when cgroup is not active
319 */
320 if (cgrp == event->cgrp)
321 __update_cgrp_time(event->cgrp);
322}
323
324static inline void
325perf_cgroup_set_timestamp(struct task_struct *task,
326 struct perf_event_context *ctx)
327{
328 struct perf_cgroup *cgrp;
329 struct perf_cgroup_info *info;
330
331 /*
332 * ctx->lock held by caller
333 * ensure we do not access cgroup data
334 * unless we have the cgroup pinned (css_get)
335 */
336 if (!task || !ctx->nr_cgroups)
337 return;
338
339 cgrp = perf_cgroup_from_task(task);
340 info = this_cpu_ptr(cgrp->info);
341 info->timestamp = ctx->timestamp;
342}
343
344#define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */
345#define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */
346
347/*
348 * reschedule events based on the cgroup constraint of task.
349 *
350 * mode SWOUT : schedule out everything
351 * mode SWIN : schedule in based on cgroup for next
352 */
353void perf_cgroup_switch(struct task_struct *task, int mode)
354{
355 struct perf_cpu_context *cpuctx;
356 struct pmu *pmu;
357 unsigned long flags;
358
359 /*
360 * disable interrupts to avoid geting nr_cgroup
361 * changes via __perf_event_disable(). Also
362 * avoids preemption.
363 */
364 local_irq_save(flags);
365
366 /*
367 * we reschedule only in the presence of cgroup
368 * constrained events.
369 */
370 rcu_read_lock();
371
372 list_for_each_entry_rcu(pmu, &pmus, entry) {
373 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
374
375 /*
376 * perf_cgroup_events says at least one
377 * context on this CPU has cgroup events.
378 *
379 * ctx->nr_cgroups reports the number of cgroup
380 * events for a context.
381 */
382 if (cpuctx->ctx.nr_cgroups > 0) {
383 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
384 perf_pmu_disable(cpuctx->ctx.pmu);
385
386 if (mode & PERF_CGROUP_SWOUT) {
387 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
388 /*
389 * must not be done before ctxswout due
390 * to event_filter_match() in event_sched_out()
391 */
392 cpuctx->cgrp = NULL;
393 }
394
395 if (mode & PERF_CGROUP_SWIN) {
396 WARN_ON_ONCE(cpuctx->cgrp);
397 /* set cgrp before ctxsw in to
398 * allow event_filter_match() to not
399 * have to pass task around
400 */
401 cpuctx->cgrp = perf_cgroup_from_task(task);
402 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
403 }
404 perf_pmu_enable(cpuctx->ctx.pmu);
405 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
406 }
407 }
408
409 rcu_read_unlock();
410
411 local_irq_restore(flags);
412}
413
414static inline void perf_cgroup_sched_out(struct task_struct *task,
415 struct task_struct *next)
416{
417 struct perf_cgroup *cgrp1;
418 struct perf_cgroup *cgrp2 = NULL;
419
420 /*
421 * we come here when we know perf_cgroup_events > 0
422 */
423 cgrp1 = perf_cgroup_from_task(task);
424
425 /*
426 * next is NULL when called from perf_event_enable_on_exec()
427 * that will systematically cause a cgroup_switch()
428 */
429 if (next)
430 cgrp2 = perf_cgroup_from_task(next);
431
432 /*
433 * only schedule out current cgroup events if we know
434 * that we are switching to a different cgroup. Otherwise,
435 * do no touch the cgroup events.
436 */
437 if (cgrp1 != cgrp2)
438 perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
439}
440
441static inline void perf_cgroup_sched_in(struct task_struct *prev,
442 struct task_struct *task)
443{
444 struct perf_cgroup *cgrp1;
445 struct perf_cgroup *cgrp2 = NULL;
446
447 /*
448 * we come here when we know perf_cgroup_events > 0
449 */
450 cgrp1 = perf_cgroup_from_task(task);
451
452 /* prev can never be NULL */
453 cgrp2 = perf_cgroup_from_task(prev);
454
455 /*
456 * only need to schedule in cgroup events if we are changing
457 * cgroup during ctxsw. Cgroup events were not scheduled
458 * out of ctxsw out if that was not the case.
459 */
460 if (cgrp1 != cgrp2)
461 perf_cgroup_switch(task, PERF_CGROUP_SWIN);
462}
463
464static inline int perf_cgroup_connect(int fd, struct perf_event *event,
465 struct perf_event_attr *attr,
466 struct perf_event *group_leader)
467{
468 struct perf_cgroup *cgrp;
469 struct cgroup_subsys_state *css;
470 struct file *file;
471 int ret = 0, fput_needed;
472
473 file = fget_light(fd, &fput_needed);
474 if (!file)
475 return -EBADF;
476
477 css = cgroup_css_from_dir(file, perf_subsys_id);
478 if (IS_ERR(css)) {
479 ret = PTR_ERR(css);
480 goto out;
481 }
482
483 cgrp = container_of(css, struct perf_cgroup, css);
484 event->cgrp = cgrp;
485
486 /* must be done before we fput() the file */
487 if (!perf_tryget_cgroup(event)) {
488 event->cgrp = NULL;
489 ret = -ENOENT;
490 goto out;
491 }
492
493 /*
494 * all events in a group must monitor
495 * the same cgroup because a task belongs
496 * to only one perf cgroup at a time
497 */
498 if (group_leader && group_leader->cgrp != cgrp) {
499 perf_detach_cgroup(event);
500 ret = -EINVAL;
501 }
502out:
503 fput_light(file, fput_needed);
504 return ret;
505}
506
507static inline void
508perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
509{
510 struct perf_cgroup_info *t;
511 t = per_cpu_ptr(event->cgrp->info, event->cpu);
512 event->shadow_ctx_time = now - t->timestamp;
513}
514
515static inline void
516perf_cgroup_defer_enabled(struct perf_event *event)
517{
518 /*
519 * when the current task's perf cgroup does not match
520 * the event's, we need to remember to call the
521 * perf_mark_enable() function the first time a task with
522 * a matching perf cgroup is scheduled in.
523 */
524 if (is_cgroup_event(event) && !perf_cgroup_match(event))
525 event->cgrp_defer_enabled = 1;
526}
527
528static inline void
529perf_cgroup_mark_enabled(struct perf_event *event,
530 struct perf_event_context *ctx)
531{
532 struct perf_event *sub;
533 u64 tstamp = perf_event_time(event);
534
535 if (!event->cgrp_defer_enabled)
536 return;
537
538 event->cgrp_defer_enabled = 0;
539
540 event->tstamp_enabled = tstamp - event->total_time_enabled;
541 list_for_each_entry(sub, &event->sibling_list, group_entry) {
542 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
543 sub->tstamp_enabled = tstamp - sub->total_time_enabled;
544 sub->cgrp_defer_enabled = 0;
545 }
546 }
547}
548#else /* !CONFIG_CGROUP_PERF */
549
550static inline bool
551perf_cgroup_match(struct perf_event *event)
552{
553 return true;
554}
555
556static inline void perf_detach_cgroup(struct perf_event *event)
557{}
558
559static inline int is_cgroup_event(struct perf_event *event)
560{
561 return 0;
562}
563
564static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
565{
566 return 0;
567}
568
569static inline void update_cgrp_time_from_event(struct perf_event *event)
570{
571}
572
573static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
574{
575}
576
577static inline void perf_cgroup_sched_out(struct task_struct *task,
578 struct task_struct *next)
579{
580}
581
582static inline void perf_cgroup_sched_in(struct task_struct *prev,
583 struct task_struct *task)
584{
585}
586
587static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
588 struct perf_event_attr *attr,
589 struct perf_event *group_leader)
590{
591 return -EINVAL;
592}
593
594static inline void
595perf_cgroup_set_timestamp(struct task_struct *task,
596 struct perf_event_context *ctx)
597{
598}
599
600void
601perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
602{
603}
604
605static inline void
606perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
607{
608}
609
610static inline u64 perf_cgroup_event_time(struct perf_event *event)
611{
612 return 0;
613}
614
615static inline void
616perf_cgroup_defer_enabled(struct perf_event *event)
617{
618}
619
620static inline void
621perf_cgroup_mark_enabled(struct perf_event *event,
622 struct perf_event_context *ctx)
623{
624}
625#endif
626
627void perf_pmu_disable(struct pmu *pmu)
628{
629 int *count = this_cpu_ptr(pmu->pmu_disable_count);
630 if (!(*count)++)
631 pmu->pmu_disable(pmu);
632}
633
634void perf_pmu_enable(struct pmu *pmu)
635{
636 int *count = this_cpu_ptr(pmu->pmu_disable_count);
637 if (!--(*count))
638 pmu->pmu_enable(pmu);
639}
640
641static DEFINE_PER_CPU(struct list_head, rotation_list);
642
643/*
644 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
645 * because they're strictly cpu affine and rotate_start is called with IRQs
646 * disabled, while rotate_context is called from IRQ context.
647 */
648static void perf_pmu_rotate_start(struct pmu *pmu)
649{
650 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
651 struct list_head *head = &__get_cpu_var(rotation_list);
652
653 WARN_ON(!irqs_disabled());
654
655 if (list_empty(&cpuctx->rotation_list))
656 list_add(&cpuctx->rotation_list, head);
657}
658
659static void get_ctx(struct perf_event_context *ctx)
660{
661 WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
662}
663
664static void put_ctx(struct perf_event_context *ctx)
665{
666 if (atomic_dec_and_test(&ctx->refcount)) {
667 if (ctx->parent_ctx)
668 put_ctx(ctx->parent_ctx);
669 if (ctx->task)
670 put_task_struct(ctx->task);
671 kfree_rcu(ctx, rcu_head);
672 }
673}
674
675static void unclone_ctx(struct perf_event_context *ctx)
676{
677 if (ctx->parent_ctx) {
678 put_ctx(ctx->parent_ctx);
679 ctx->parent_ctx = NULL;
680 }
681}
682
683static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
684{
685 /*
686 * only top level events have the pid namespace they were created in
687 */
688 if (event->parent)
689 event = event->parent;
690
691 return task_tgid_nr_ns(p, event->ns);
692}
693
694static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
695{
696 /*
697 * only top level events have the pid namespace they were created in
698 */
699 if (event->parent)
700 event = event->parent;
701
702 return task_pid_nr_ns(p, event->ns);
703}
704
705/*
706 * If we inherit events we want to return the parent event id
707 * to userspace.
708 */
709static u64 primary_event_id(struct perf_event *event)
710{
711 u64 id = event->id;
712
713 if (event->parent)
714 id = event->parent->id;
715
716 return id;
717}
718
719/*
720 * Get the perf_event_context for a task and lock it.
721 * This has to cope with with the fact that until it is locked,
722 * the context could get moved to another task.
723 */
724static struct perf_event_context *
725perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
726{
727 struct perf_event_context *ctx;
728
729 rcu_read_lock();
730retry:
731 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
732 if (ctx) {
733 /*
734 * If this context is a clone of another, it might
735 * get swapped for another underneath us by
736 * perf_event_task_sched_out, though the
737 * rcu_read_lock() protects us from any context
738 * getting freed. Lock the context and check if it
739 * got swapped before we could get the lock, and retry
740 * if so. If we locked the right context, then it
741 * can't get swapped on us any more.
742 */
743 raw_spin_lock_irqsave(&ctx->lock, *flags);
744 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
745 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
746 goto retry;
747 }
748
749 if (!atomic_inc_not_zero(&ctx->refcount)) {
750 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
751 ctx = NULL;
752 }
753 }
754 rcu_read_unlock();
755 return ctx;
756}
757
758/*
759 * Get the context for a task and increment its pin_count so it
760 * can't get swapped to another task. This also increments its
761 * reference count so that the context can't get freed.
762 */
763static struct perf_event_context *
764perf_pin_task_context(struct task_struct *task, int ctxn)
765{
766 struct perf_event_context *ctx;
767 unsigned long flags;
768
769 ctx = perf_lock_task_context(task, ctxn, &flags);
770 if (ctx) {
771 ++ctx->pin_count;
772 raw_spin_unlock_irqrestore(&ctx->lock, flags);
773 }
774 return ctx;
775}
776
777static void perf_unpin_context(struct perf_event_context *ctx)
778{
779 unsigned long flags;
780
781 raw_spin_lock_irqsave(&ctx->lock, flags);
782 --ctx->pin_count;
783 raw_spin_unlock_irqrestore(&ctx->lock, flags);
784}
785
786/*
787 * Update the record of the current time in a context.
788 */
789static void update_context_time(struct perf_event_context *ctx)
790{
791 u64 now = perf_clock();
792
793 ctx->time += now - ctx->timestamp;
794 ctx->timestamp = now;
795}
796
797static u64 perf_event_time(struct perf_event *event)
798{
799 struct perf_event_context *ctx = event->ctx;
800
801 if (is_cgroup_event(event))
802 return perf_cgroup_event_time(event);
803
804 return ctx ? ctx->time : 0;
805}
806
807/*
808 * Update the total_time_enabled and total_time_running fields for a event.
809 * The caller of this function needs to hold the ctx->lock.
810 */
811static void update_event_times(struct perf_event *event)
812{
813 struct perf_event_context *ctx = event->ctx;
814 u64 run_end;
815
816 if (event->state < PERF_EVENT_STATE_INACTIVE ||
817 event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
818 return;
819 /*
820 * in cgroup mode, time_enabled represents
821 * the time the event was enabled AND active
822 * tasks were in the monitored cgroup. This is
823 * independent of the activity of the context as
824 * there may be a mix of cgroup and non-cgroup events.
825 *
826 * That is why we treat cgroup events differently
827 * here.
828 */
829 if (is_cgroup_event(event))
830 run_end = perf_cgroup_event_time(event);
831 else if (ctx->is_active)
832 run_end = ctx->time;
833 else
834 run_end = event->tstamp_stopped;
835
836 event->total_time_enabled = run_end - event->tstamp_enabled;
837
838 if (event->state == PERF_EVENT_STATE_INACTIVE)
839 run_end = event->tstamp_stopped;
840 else
841 run_end = perf_event_time(event);
842
843 event->total_time_running = run_end - event->tstamp_running;
844
845}
846
847/*
848 * Update total_time_enabled and total_time_running for all events in a group.
849 */
850static void update_group_times(struct perf_event *leader)
851{
852 struct perf_event *event;
853
854 update_event_times(leader);
855 list_for_each_entry(event, &leader->sibling_list, group_entry)
856 update_event_times(event);
857}
858
859static struct list_head *
860ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
861{
862 if (event->attr.pinned)
863 return &ctx->pinned_groups;
864 else
865 return &ctx->flexible_groups;
866}
867
868/*
869 * Add a event from the lists for its context.
870 * Must be called with ctx->mutex and ctx->lock held.
871 */
872static void
873list_add_event(struct perf_event *event, struct perf_event_context *ctx)
874{
875 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
876 event->attach_state |= PERF_ATTACH_CONTEXT;
877
878 /*
879 * If we're a stand alone event or group leader, we go to the context
880 * list, group events are kept attached to the group so that
881 * perf_group_detach can, at all times, locate all siblings.
882 */
883 if (event->group_leader == event) {
884 struct list_head *list;
885
886 if (is_software_event(event))
887 event->group_flags |= PERF_GROUP_SOFTWARE;
888
889 list = ctx_group_list(event, ctx);
890 list_add_tail(&event->group_entry, list);
891 }
892
893 if (is_cgroup_event(event))
894 ctx->nr_cgroups++;
895
896 if (has_branch_stack(event))
897 ctx->nr_branch_stack++;
898
899 list_add_rcu(&event->event_entry, &ctx->event_list);
900 if (!ctx->nr_events)
901 perf_pmu_rotate_start(ctx->pmu);
902 ctx->nr_events++;
903 if (event->attr.inherit_stat)
904 ctx->nr_stat++;
905}
906
907/*
908 * Called at perf_event creation and when events are attached/detached from a
909 * group.
910 */
911static void perf_event__read_size(struct perf_event *event)
912{
913 int entry = sizeof(u64); /* value */
914 int size = 0;
915 int nr = 1;
916
917 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
918 size += sizeof(u64);
919
920 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
921 size += sizeof(u64);
922
923 if (event->attr.read_format & PERF_FORMAT_ID)
924 entry += sizeof(u64);
925
926 if (event->attr.read_format & PERF_FORMAT_GROUP) {
927 nr += event->group_leader->nr_siblings;
928 size += sizeof(u64);
929 }
930
931 size += entry * nr;
932 event->read_size = size;
933}
934
935static void perf_event__header_size(struct perf_event *event)
936{
937 struct perf_sample_data *data;
938 u64 sample_type = event->attr.sample_type;
939 u16 size = 0;
940
941 perf_event__read_size(event);
942
943 if (sample_type & PERF_SAMPLE_IP)
944 size += sizeof(data->ip);
945
946 if (sample_type & PERF_SAMPLE_ADDR)
947 size += sizeof(data->addr);
948
949 if (sample_type & PERF_SAMPLE_PERIOD)
950 size += sizeof(data->period);
951
952 if (sample_type & PERF_SAMPLE_READ)
953 size += event->read_size;
954
955 event->header_size = size;
956}
957
958static void perf_event__id_header_size(struct perf_event *event)
959{
960 struct perf_sample_data *data;
961 u64 sample_type = event->attr.sample_type;
962 u16 size = 0;
963
964 if (sample_type & PERF_SAMPLE_TID)
965 size += sizeof(data->tid_entry);
966
967 if (sample_type & PERF_SAMPLE_TIME)
968 size += sizeof(data->time);
969
970 if (sample_type & PERF_SAMPLE_ID)
971 size += sizeof(data->id);
972
973 if (sample_type & PERF_SAMPLE_STREAM_ID)
974 size += sizeof(data->stream_id);
975
976 if (sample_type & PERF_SAMPLE_CPU)
977 size += sizeof(data->cpu_entry);
978
979 event->id_header_size = size;
980}
981
982static void perf_group_attach(struct perf_event *event)
983{
984 struct perf_event *group_leader = event->group_leader, *pos;
985
986 /*
987 * We can have double attach due to group movement in perf_event_open.
988 */
989 if (event->attach_state & PERF_ATTACH_GROUP)
990 return;
991
992 event->attach_state |= PERF_ATTACH_GROUP;
993
994 if (group_leader == event)
995 return;
996
997 if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
998 !is_software_event(event))
999 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1000
1001 list_add_tail(&event->group_entry, &group_leader->sibling_list);
1002 group_leader->nr_siblings++;
1003
1004 perf_event__header_size(group_leader);
1005
1006 list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1007 perf_event__header_size(pos);
1008}
1009
1010/*
1011 * Remove a event from the lists for its context.
1012 * Must be called with ctx->mutex and ctx->lock held.
1013 */
1014static void
1015list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1016{
1017 struct perf_cpu_context *cpuctx;
1018 /*
1019 * We can have double detach due to exit/hot-unplug + close.
1020 */
1021 if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1022 return;
1023
1024 event->attach_state &= ~PERF_ATTACH_CONTEXT;
1025
1026 if (is_cgroup_event(event)) {
1027 ctx->nr_cgroups--;
1028 cpuctx = __get_cpu_context(ctx);
1029 /*
1030 * if there are no more cgroup events
1031 * then cler cgrp to avoid stale pointer
1032 * in update_cgrp_time_from_cpuctx()
1033 */
1034 if (!ctx->nr_cgroups)
1035 cpuctx->cgrp = NULL;
1036 }
1037
1038 if (has_branch_stack(event))
1039 ctx->nr_branch_stack--;
1040
1041 ctx->nr_events--;
1042 if (event->attr.inherit_stat)
1043 ctx->nr_stat--;
1044
1045 list_del_rcu(&event->event_entry);
1046
1047 if (event->group_leader == event)
1048 list_del_init(&event->group_entry);
1049
1050 update_group_times(event);
1051
1052 /*
1053 * If event was in error state, then keep it
1054 * that way, otherwise bogus counts will be
1055 * returned on read(). The only way to get out
1056 * of error state is by explicit re-enabling
1057 * of the event
1058 */
1059 if (event->state > PERF_EVENT_STATE_OFF)
1060 event->state = PERF_EVENT_STATE_OFF;
1061}
1062
1063static void perf_group_detach(struct perf_event *event)
1064{
1065 struct perf_event *sibling, *tmp;
1066 struct list_head *list = NULL;
1067
1068 /*
1069 * We can have double detach due to exit/hot-unplug + close.
1070 */
1071 if (!(event->attach_state & PERF_ATTACH_GROUP))
1072 return;
1073
1074 event->attach_state &= ~PERF_ATTACH_GROUP;
1075
1076 /*
1077 * If this is a sibling, remove it from its group.
1078 */
1079 if (event->group_leader != event) {
1080 list_del_init(&event->group_entry);
1081 event->group_leader->nr_siblings--;
1082 goto out;
1083 }
1084
1085 if (!list_empty(&event->group_entry))
1086 list = &event->group_entry;
1087
1088 /*
1089 * If this was a group event with sibling events then
1090 * upgrade the siblings to singleton events by adding them
1091 * to whatever list we are on.
1092 */
1093 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1094 if (list)
1095 list_move_tail(&sibling->group_entry, list);
1096 sibling->group_leader = sibling;
1097
1098 /* Inherit group flags from the previous leader */
1099 sibling->group_flags = event->group_flags;
1100 }
1101
1102out:
1103 perf_event__header_size(event->group_leader);
1104
1105 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1106 perf_event__header_size(tmp);
1107}
1108
1109static inline int
1110event_filter_match(struct perf_event *event)
1111{
1112 return (event->cpu == -1 || event->cpu == smp_processor_id())
1113 && perf_cgroup_match(event);
1114}
1115
1116static void
1117event_sched_out(struct perf_event *event,
1118 struct perf_cpu_context *cpuctx,
1119 struct perf_event_context *ctx)
1120{
1121 u64 tstamp = perf_event_time(event);
1122 u64 delta;
1123 /*
1124 * An event which could not be activated because of
1125 * filter mismatch still needs to have its timings
1126 * maintained, otherwise bogus information is return
1127 * via read() for time_enabled, time_running:
1128 */
1129 if (event->state == PERF_EVENT_STATE_INACTIVE
1130 && !event_filter_match(event)) {
1131 delta = tstamp - event->tstamp_stopped;
1132 event->tstamp_running += delta;
1133 event->tstamp_stopped = tstamp;
1134 }
1135
1136 if (event->state != PERF_EVENT_STATE_ACTIVE)
1137 return;
1138
1139 event->state = PERF_EVENT_STATE_INACTIVE;
1140 if (event->pending_disable) {
1141 event->pending_disable = 0;
1142 event->state = PERF_EVENT_STATE_OFF;
1143 }
1144 event->tstamp_stopped = tstamp;
1145 event->pmu->del(event, 0);
1146 event->oncpu = -1;
1147
1148 if (!is_software_event(event))
1149 cpuctx->active_oncpu--;
1150 ctx->nr_active--;
1151 if (event->attr.freq && event->attr.sample_freq)
1152 ctx->nr_freq--;
1153 if (event->attr.exclusive || !cpuctx->active_oncpu)
1154 cpuctx->exclusive = 0;
1155}
1156
1157static void
1158group_sched_out(struct perf_event *group_event,
1159 struct perf_cpu_context *cpuctx,
1160 struct perf_event_context *ctx)
1161{
1162 struct perf_event *event;
1163 int state = group_event->state;
1164
1165 event_sched_out(group_event, cpuctx, ctx);
1166
1167 /*
1168 * Schedule out siblings (if any):
1169 */
1170 list_for_each_entry(event, &group_event->sibling_list, group_entry)
1171 event_sched_out(event, cpuctx, ctx);
1172
1173 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1174 cpuctx->exclusive = 0;
1175}
1176
1177/*
1178 * Cross CPU call to remove a performance event
1179 *
1180 * We disable the event on the hardware level first. After that we
1181 * remove it from the context list.
1182 */
1183static int __perf_remove_from_context(void *info)
1184{
1185 struct perf_event *event = info;
1186 struct perf_event_context *ctx = event->ctx;
1187 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1188
1189 raw_spin_lock(&ctx->lock);
1190 event_sched_out(event, cpuctx, ctx);
1191 list_del_event(event, ctx);
1192 if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1193 ctx->is_active = 0;
1194 cpuctx->task_ctx = NULL;
1195 }
1196 raw_spin_unlock(&ctx->lock);
1197
1198 return 0;
1199}
1200
1201
1202/*
1203 * Remove the event from a task's (or a CPU's) list of events.
1204 *
1205 * CPU events are removed with a smp call. For task events we only
1206 * call when the task is on a CPU.
1207 *
1208 * If event->ctx is a cloned context, callers must make sure that
1209 * every task struct that event->ctx->task could possibly point to
1210 * remains valid. This is OK when called from perf_release since
1211 * that only calls us on the top-level context, which can't be a clone.
1212 * When called from perf_event_exit_task, it's OK because the
1213 * context has been detached from its task.
1214 */
1215static void perf_remove_from_context(struct perf_event *event)
1216{
1217 struct perf_event_context *ctx = event->ctx;
1218 struct task_struct *task = ctx->task;
1219
1220 lockdep_assert_held(&ctx->mutex);
1221
1222 if (!task) {
1223 /*
1224 * Per cpu events are removed via an smp call and
1225 * the removal is always successful.
1226 */
1227 cpu_function_call(event->cpu, __perf_remove_from_context, event);
1228 return;
1229 }
1230
1231retry:
1232 if (!task_function_call(task, __perf_remove_from_context, event))
1233 return;
1234
1235 raw_spin_lock_irq(&ctx->lock);
1236 /*
1237 * If we failed to find a running task, but find the context active now
1238 * that we've acquired the ctx->lock, retry.
1239 */
1240 if (ctx->is_active) {
1241 raw_spin_unlock_irq(&ctx->lock);
1242 goto retry;
1243 }
1244
1245 /*
1246 * Since the task isn't running, its safe to remove the event, us
1247 * holding the ctx->lock ensures the task won't get scheduled in.
1248 */
1249 list_del_event(event, ctx);
1250 raw_spin_unlock_irq(&ctx->lock);
1251}
1252
1253/*
1254 * Cross CPU call to disable a performance event
1255 */
1256static int __perf_event_disable(void *info)
1257{
1258 struct perf_event *event = info;
1259 struct perf_event_context *ctx = event->ctx;
1260 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1261
1262 /*
1263 * If this is a per-task event, need to check whether this
1264 * event's task is the current task on this cpu.
1265 *
1266 * Can trigger due to concurrent perf_event_context_sched_out()
1267 * flipping contexts around.
1268 */
1269 if (ctx->task && cpuctx->task_ctx != ctx)
1270 return -EINVAL;
1271
1272 raw_spin_lock(&ctx->lock);
1273
1274 /*
1275 * If the event is on, turn it off.
1276 * If it is in error state, leave it in error state.
1277 */
1278 if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1279 update_context_time(ctx);
1280 update_cgrp_time_from_event(event);
1281 update_group_times(event);
1282 if (event == event->group_leader)
1283 group_sched_out(event, cpuctx, ctx);
1284 else
1285 event_sched_out(event, cpuctx, ctx);
1286 event->state = PERF_EVENT_STATE_OFF;
1287 }
1288
1289 raw_spin_unlock(&ctx->lock);
1290
1291 return 0;
1292}
1293
1294/*
1295 * Disable a event.
1296 *
1297 * If event->ctx is a cloned context, callers must make sure that
1298 * every task struct that event->ctx->task could possibly point to
1299 * remains valid. This condition is satisifed when called through
1300 * perf_event_for_each_child or perf_event_for_each because they
1301 * hold the top-level event's child_mutex, so any descendant that
1302 * goes to exit will block in sync_child_event.
1303 * When called from perf_pending_event it's OK because event->ctx
1304 * is the current context on this CPU and preemption is disabled,
1305 * hence we can't get into perf_event_task_sched_out for this context.
1306 */
1307void perf_event_disable(struct perf_event *event)
1308{
1309 struct perf_event_context *ctx = event->ctx;
1310 struct task_struct *task = ctx->task;
1311
1312 if (!task) {
1313 /*
1314 * Disable the event on the cpu that it's on
1315 */
1316 cpu_function_call(event->cpu, __perf_event_disable, event);
1317 return;
1318 }
1319
1320retry:
1321 if (!task_function_call(task, __perf_event_disable, event))
1322 return;
1323
1324 raw_spin_lock_irq(&ctx->lock);
1325 /*
1326 * If the event is still active, we need to retry the cross-call.
1327 */
1328 if (event->state == PERF_EVENT_STATE_ACTIVE) {
1329 raw_spin_unlock_irq(&ctx->lock);
1330 /*
1331 * Reload the task pointer, it might have been changed by
1332 * a concurrent perf_event_context_sched_out().
1333 */
1334 task = ctx->task;
1335 goto retry;
1336 }
1337
1338 /*
1339 * Since we have the lock this context can't be scheduled
1340 * in, so we can change the state safely.
1341 */
1342 if (event->state == PERF_EVENT_STATE_INACTIVE) {
1343 update_group_times(event);
1344 event->state = PERF_EVENT_STATE_OFF;
1345 }
1346 raw_spin_unlock_irq(&ctx->lock);
1347}
1348EXPORT_SYMBOL_GPL(perf_event_disable);
1349
1350static void perf_set_shadow_time(struct perf_event *event,
1351 struct perf_event_context *ctx,
1352 u64 tstamp)
1353{
1354 /*
1355 * use the correct time source for the time snapshot
1356 *
1357 * We could get by without this by leveraging the
1358 * fact that to get to this function, the caller
1359 * has most likely already called update_context_time()
1360 * and update_cgrp_time_xx() and thus both timestamp
1361 * are identical (or very close). Given that tstamp is,
1362 * already adjusted for cgroup, we could say that:
1363 * tstamp - ctx->timestamp
1364 * is equivalent to
1365 * tstamp - cgrp->timestamp.
1366 *
1367 * Then, in perf_output_read(), the calculation would
1368 * work with no changes because:
1369 * - event is guaranteed scheduled in
1370 * - no scheduled out in between
1371 * - thus the timestamp would be the same
1372 *
1373 * But this is a bit hairy.
1374 *
1375 * So instead, we have an explicit cgroup call to remain
1376 * within the time time source all along. We believe it
1377 * is cleaner and simpler to understand.
1378 */
1379 if (is_cgroup_event(event))
1380 perf_cgroup_set_shadow_time(event, tstamp);
1381 else
1382 event->shadow_ctx_time = tstamp - ctx->timestamp;
1383}
1384
1385#define MAX_INTERRUPTS (~0ULL)
1386
1387static void perf_log_throttle(struct perf_event *event, int enable);
1388
1389static int
1390event_sched_in(struct perf_event *event,
1391 struct perf_cpu_context *cpuctx,
1392 struct perf_event_context *ctx)
1393{
1394 u64 tstamp = perf_event_time(event);
1395
1396 if (event->state <= PERF_EVENT_STATE_OFF)
1397 return 0;
1398
1399 event->state = PERF_EVENT_STATE_ACTIVE;
1400 event->oncpu = smp_processor_id();
1401
1402 /*
1403 * Unthrottle events, since we scheduled we might have missed several
1404 * ticks already, also for a heavily scheduling task there is little
1405 * guarantee it'll get a tick in a timely manner.
1406 */
1407 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1408 perf_log_throttle(event, 1);
1409 event->hw.interrupts = 0;
1410 }
1411
1412 /*
1413 * The new state must be visible before we turn it on in the hardware:
1414 */
1415 smp_wmb();
1416
1417 if (event->pmu->add(event, PERF_EF_START)) {
1418 event->state = PERF_EVENT_STATE_INACTIVE;
1419 event->oncpu = -1;
1420 return -EAGAIN;
1421 }
1422
1423 event->tstamp_running += tstamp - event->tstamp_stopped;
1424
1425 perf_set_shadow_time(event, ctx, tstamp);
1426
1427 if (!is_software_event(event))
1428 cpuctx->active_oncpu++;
1429 ctx->nr_active++;
1430 if (event->attr.freq && event->attr.sample_freq)
1431 ctx->nr_freq++;
1432
1433 if (event->attr.exclusive)
1434 cpuctx->exclusive = 1;
1435
1436 return 0;
1437}
1438
1439static int
1440group_sched_in(struct perf_event *group_event,
1441 struct perf_cpu_context *cpuctx,
1442 struct perf_event_context *ctx)
1443{
1444 struct perf_event *event, *partial_group = NULL;
1445 struct pmu *pmu = group_event->pmu;
1446 u64 now = ctx->time;
1447 bool simulate = false;
1448
1449 if (group_event->state == PERF_EVENT_STATE_OFF)
1450 return 0;
1451
1452 pmu->start_txn(pmu);
1453
1454 if (event_sched_in(group_event, cpuctx, ctx)) {
1455 pmu->cancel_txn(pmu);
1456 return -EAGAIN;
1457 }
1458
1459 /*
1460 * Schedule in siblings as one group (if any):
1461 */
1462 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1463 if (event_sched_in(event, cpuctx, ctx)) {
1464 partial_group = event;
1465 goto group_error;
1466 }
1467 }
1468
1469 if (!pmu->commit_txn(pmu))
1470 return 0;
1471
1472group_error:
1473 /*
1474 * Groups can be scheduled in as one unit only, so undo any
1475 * partial group before returning:
1476 * The events up to the failed event are scheduled out normally,
1477 * tstamp_stopped will be updated.
1478 *
1479 * The failed events and the remaining siblings need to have
1480 * their timings updated as if they had gone thru event_sched_in()
1481 * and event_sched_out(). This is required to get consistent timings
1482 * across the group. This also takes care of the case where the group
1483 * could never be scheduled by ensuring tstamp_stopped is set to mark
1484 * the time the event was actually stopped, such that time delta
1485 * calculation in update_event_times() is correct.
1486 */
1487 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1488 if (event == partial_group)
1489 simulate = true;
1490
1491 if (simulate) {
1492 event->tstamp_running += now - event->tstamp_stopped;
1493 event->tstamp_stopped = now;
1494 } else {
1495 event_sched_out(event, cpuctx, ctx);
1496 }
1497 }
1498 event_sched_out(group_event, cpuctx, ctx);
1499
1500 pmu->cancel_txn(pmu);
1501
1502 return -EAGAIN;
1503}
1504
1505/*
1506 * Work out whether we can put this event group on the CPU now.
1507 */
1508static int group_can_go_on(struct perf_event *event,
1509 struct perf_cpu_context *cpuctx,
1510 int can_add_hw)
1511{
1512 /*
1513 * Groups consisting entirely of software events can always go on.
1514 */
1515 if (event->group_flags & PERF_GROUP_SOFTWARE)
1516 return 1;
1517 /*
1518 * If an exclusive group is already on, no other hardware
1519 * events can go on.
1520 */
1521 if (cpuctx->exclusive)
1522 return 0;
1523 /*
1524 * If this group is exclusive and there are already
1525 * events on the CPU, it can't go on.
1526 */
1527 if (event->attr.exclusive && cpuctx->active_oncpu)
1528 return 0;
1529 /*
1530 * Otherwise, try to add it if all previous groups were able
1531 * to go on.
1532 */
1533 return can_add_hw;
1534}
1535
1536static void add_event_to_ctx(struct perf_event *event,
1537 struct perf_event_context *ctx)
1538{
1539 u64 tstamp = perf_event_time(event);
1540
1541 list_add_event(event, ctx);
1542 perf_group_attach(event);
1543 event->tstamp_enabled = tstamp;
1544 event->tstamp_running = tstamp;
1545 event->tstamp_stopped = tstamp;
1546}
1547
1548static void task_ctx_sched_out(struct perf_event_context *ctx);
1549static void
1550ctx_sched_in(struct perf_event_context *ctx,
1551 struct perf_cpu_context *cpuctx,
1552 enum event_type_t event_type,
1553 struct task_struct *task);
1554
1555static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1556 struct perf_event_context *ctx,
1557 struct task_struct *task)
1558{
1559 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1560 if (ctx)
1561 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1562 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1563 if (ctx)
1564 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1565}
1566
1567/*
1568 * Cross CPU call to install and enable a performance event
1569 *
1570 * Must be called with ctx->mutex held
1571 */
1572static int __perf_install_in_context(void *info)
1573{
1574 struct perf_event *event = info;
1575 struct perf_event_context *ctx = event->ctx;
1576 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1577 struct perf_event_context *task_ctx = cpuctx->task_ctx;
1578 struct task_struct *task = current;
1579
1580 perf_ctx_lock(cpuctx, task_ctx);
1581 perf_pmu_disable(cpuctx->ctx.pmu);
1582
1583 /*
1584 * If there was an active task_ctx schedule it out.
1585 */
1586 if (task_ctx)
1587 task_ctx_sched_out(task_ctx);
1588
1589 /*
1590 * If the context we're installing events in is not the
1591 * active task_ctx, flip them.
1592 */
1593 if (ctx->task && task_ctx != ctx) {
1594 if (task_ctx)
1595 raw_spin_unlock(&task_ctx->lock);
1596 raw_spin_lock(&ctx->lock);
1597 task_ctx = ctx;
1598 }
1599
1600 if (task_ctx) {
1601 cpuctx->task_ctx = task_ctx;
1602 task = task_ctx->task;
1603 }
1604
1605 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1606
1607 update_context_time(ctx);
1608 /*
1609 * update cgrp time only if current cgrp
1610 * matches event->cgrp. Must be done before
1611 * calling add_event_to_ctx()
1612 */
1613 update_cgrp_time_from_event(event);
1614
1615 add_event_to_ctx(event, ctx);
1616
1617 /*
1618 * Schedule everything back in
1619 */
1620 perf_event_sched_in(cpuctx, task_ctx, task);
1621
1622 perf_pmu_enable(cpuctx->ctx.pmu);
1623 perf_ctx_unlock(cpuctx, task_ctx);
1624
1625 return 0;
1626}
1627
1628/*
1629 * Attach a performance event to a context
1630 *
1631 * First we add the event to the list with the hardware enable bit
1632 * in event->hw_config cleared.
1633 *
1634 * If the event is attached to a task which is on a CPU we use a smp
1635 * call to enable it in the task context. The task might have been
1636 * scheduled away, but we check this in the smp call again.
1637 */
1638static void
1639perf_install_in_context(struct perf_event_context *ctx,
1640 struct perf_event *event,
1641 int cpu)
1642{
1643 struct task_struct *task = ctx->task;
1644
1645 lockdep_assert_held(&ctx->mutex);
1646
1647 event->ctx = ctx;
1648
1649 if (!task) {
1650 /*
1651 * Per cpu events are installed via an smp call and
1652 * the install is always successful.
1653 */
1654 cpu_function_call(cpu, __perf_install_in_context, event);
1655 return;
1656 }
1657
1658retry:
1659 if (!task_function_call(task, __perf_install_in_context, event))
1660 return;
1661
1662 raw_spin_lock_irq(&ctx->lock);
1663 /*
1664 * If we failed to find a running task, but find the context active now
1665 * that we've acquired the ctx->lock, retry.
1666 */
1667 if (ctx->is_active) {
1668 raw_spin_unlock_irq(&ctx->lock);
1669 goto retry;
1670 }
1671
1672 /*
1673 * Since the task isn't running, its safe to add the event, us holding
1674 * the ctx->lock ensures the task won't get scheduled in.
1675 */
1676 add_event_to_ctx(event, ctx);
1677 raw_spin_unlock_irq(&ctx->lock);
1678}
1679
1680/*
1681 * Put a event into inactive state and update time fields.
1682 * Enabling the leader of a group effectively enables all
1683 * the group members that aren't explicitly disabled, so we
1684 * have to update their ->tstamp_enabled also.
1685 * Note: this works for group members as well as group leaders
1686 * since the non-leader members' sibling_lists will be empty.
1687 */
1688static void __perf_event_mark_enabled(struct perf_event *event)
1689{
1690 struct perf_event *sub;
1691 u64 tstamp = perf_event_time(event);
1692
1693 event->state = PERF_EVENT_STATE_INACTIVE;
1694 event->tstamp_enabled = tstamp - event->total_time_enabled;
1695 list_for_each_entry(sub, &event->sibling_list, group_entry) {
1696 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1697 sub->tstamp_enabled = tstamp - sub->total_time_enabled;
1698 }
1699}
1700
1701/*
1702 * Cross CPU call to enable a performance event
1703 */
1704static int __perf_event_enable(void *info)
1705{
1706 struct perf_event *event = info;
1707 struct perf_event_context *ctx = event->ctx;
1708 struct perf_event *leader = event->group_leader;
1709 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1710 int err;
1711
1712 if (WARN_ON_ONCE(!ctx->is_active))
1713 return -EINVAL;
1714
1715 raw_spin_lock(&ctx->lock);
1716 update_context_time(ctx);
1717
1718 if (event->state >= PERF_EVENT_STATE_INACTIVE)
1719 goto unlock;
1720
1721 /*
1722 * set current task's cgroup time reference point
1723 */
1724 perf_cgroup_set_timestamp(current, ctx);
1725
1726 __perf_event_mark_enabled(event);
1727
1728 if (!event_filter_match(event)) {
1729 if (is_cgroup_event(event))
1730 perf_cgroup_defer_enabled(event);
1731 goto unlock;
1732 }
1733
1734 /*
1735 * If the event is in a group and isn't the group leader,
1736 * then don't put it on unless the group is on.
1737 */
1738 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1739 goto unlock;
1740
1741 if (!group_can_go_on(event, cpuctx, 1)) {
1742 err = -EEXIST;
1743 } else {
1744 if (event == leader)
1745 err = group_sched_in(event, cpuctx, ctx);
1746 else
1747 err = event_sched_in(event, cpuctx, ctx);
1748 }
1749
1750 if (err) {
1751 /*
1752 * If this event can't go on and it's part of a
1753 * group, then the whole group has to come off.
1754 */
1755 if (leader != event)
1756 group_sched_out(leader, cpuctx, ctx);
1757 if (leader->attr.pinned) {
1758 update_group_times(leader);
1759 leader->state = PERF_EVENT_STATE_ERROR;
1760 }
1761 }
1762
1763unlock:
1764 raw_spin_unlock(&ctx->lock);
1765
1766 return 0;
1767}
1768
1769/*
1770 * Enable a event.
1771 *
1772 * If event->ctx is a cloned context, callers must make sure that
1773 * every task struct that event->ctx->task could possibly point to
1774 * remains valid. This condition is satisfied when called through
1775 * perf_event_for_each_child or perf_event_for_each as described
1776 * for perf_event_disable.
1777 */
1778void perf_event_enable(struct perf_event *event)
1779{
1780 struct perf_event_context *ctx = event->ctx;
1781 struct task_struct *task = ctx->task;
1782
1783 if (!task) {
1784 /*
1785 * Enable the event on the cpu that it's on
1786 */
1787 cpu_function_call(event->cpu, __perf_event_enable, event);
1788 return;
1789 }
1790
1791 raw_spin_lock_irq(&ctx->lock);
1792 if (event->state >= PERF_EVENT_STATE_INACTIVE)
1793 goto out;
1794
1795 /*
1796 * If the event is in error state, clear that first.
1797 * That way, if we see the event in error state below, we
1798 * know that it has gone back into error state, as distinct
1799 * from the task having been scheduled away before the
1800 * cross-call arrived.
1801 */
1802 if (event->state == PERF_EVENT_STATE_ERROR)
1803 event->state = PERF_EVENT_STATE_OFF;
1804
1805retry:
1806 if (!ctx->is_active) {
1807 __perf_event_mark_enabled(event);
1808 goto out;
1809 }
1810
1811 raw_spin_unlock_irq(&ctx->lock);
1812
1813 if (!task_function_call(task, __perf_event_enable, event))
1814 return;
1815
1816 raw_spin_lock_irq(&ctx->lock);
1817
1818 /*
1819 * If the context is active and the event is still off,
1820 * we need to retry the cross-call.
1821 */
1822 if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
1823 /*
1824 * task could have been flipped by a concurrent
1825 * perf_event_context_sched_out()
1826 */
1827 task = ctx->task;
1828 goto retry;
1829 }
1830
1831out:
1832 raw_spin_unlock_irq(&ctx->lock);
1833}
1834EXPORT_SYMBOL_GPL(perf_event_enable);
1835
1836int perf_event_refresh(struct perf_event *event, int refresh)
1837{
1838 /*
1839 * not supported on inherited events
1840 */
1841 if (event->attr.inherit || !is_sampling_event(event))
1842 return -EINVAL;
1843
1844 atomic_add(refresh, &event->event_limit);
1845 perf_event_enable(event);
1846
1847 return 0;
1848}
1849EXPORT_SYMBOL_GPL(perf_event_refresh);
1850
1851static void ctx_sched_out(struct perf_event_context *ctx,
1852 struct perf_cpu_context *cpuctx,
1853 enum event_type_t event_type)
1854{
1855 struct perf_event *event;
1856 int is_active = ctx->is_active;
1857
1858 ctx->is_active &= ~event_type;
1859 if (likely(!ctx->nr_events))
1860 return;
1861
1862 update_context_time(ctx);
1863 update_cgrp_time_from_cpuctx(cpuctx);
1864 if (!ctx->nr_active)
1865 return;
1866
1867 perf_pmu_disable(ctx->pmu);
1868 if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
1869 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1870 group_sched_out(event, cpuctx, ctx);
1871 }
1872
1873 if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
1874 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1875 group_sched_out(event, cpuctx, ctx);
1876 }
1877 perf_pmu_enable(ctx->pmu);
1878}
1879
1880/*
1881 * Test whether two contexts are equivalent, i.e. whether they
1882 * have both been cloned from the same version of the same context
1883 * and they both have the same number of enabled events.
1884 * If the number of enabled events is the same, then the set
1885 * of enabled events should be the same, because these are both
1886 * inherited contexts, therefore we can't access individual events
1887 * in them directly with an fd; we can only enable/disable all
1888 * events via prctl, or enable/disable all events in a family
1889 * via ioctl, which will have the same effect on both contexts.
1890 */
1891static int context_equiv(struct perf_event_context *ctx1,
1892 struct perf_event_context *ctx2)
1893{
1894 return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1895 && ctx1->parent_gen == ctx2->parent_gen
1896 && !ctx1->pin_count && !ctx2->pin_count;
1897}
1898
1899static void __perf_event_sync_stat(struct perf_event *event,
1900 struct perf_event *next_event)
1901{
1902 u64 value;
1903
1904 if (!event->attr.inherit_stat)
1905 return;
1906
1907 /*
1908 * Update the event value, we cannot use perf_event_read()
1909 * because we're in the middle of a context switch and have IRQs
1910 * disabled, which upsets smp_call_function_single(), however
1911 * we know the event must be on the current CPU, therefore we
1912 * don't need to use it.
1913 */
1914 switch (event->state) {
1915 case PERF_EVENT_STATE_ACTIVE:
1916 event->pmu->read(event);
1917 /* fall-through */
1918
1919 case PERF_EVENT_STATE_INACTIVE:
1920 update_event_times(event);
1921 break;
1922
1923 default:
1924 break;
1925 }
1926
1927 /*
1928 * In order to keep per-task stats reliable we need to flip the event
1929 * values when we flip the contexts.
1930 */
1931 value = local64_read(&next_event->count);
1932 value = local64_xchg(&event->count, value);
1933 local64_set(&next_event->count, value);
1934
1935 swap(event->total_time_enabled, next_event->total_time_enabled);
1936 swap(event->total_time_running, next_event->total_time_running);
1937
1938 /*
1939 * Since we swizzled the values, update the user visible data too.
1940 */
1941 perf_event_update_userpage(event);
1942 perf_event_update_userpage(next_event);
1943}
1944
1945#define list_next_entry(pos, member) \
1946 list_entry(pos->member.next, typeof(*pos), member)
1947
1948static void perf_event_sync_stat(struct perf_event_context *ctx,
1949 struct perf_event_context *next_ctx)
1950{
1951 struct perf_event *event, *next_event;
1952
1953 if (!ctx->nr_stat)
1954 return;
1955
1956 update_context_time(ctx);
1957
1958 event = list_first_entry(&ctx->event_list,
1959 struct perf_event, event_entry);
1960
1961 next_event = list_first_entry(&next_ctx->event_list,
1962 struct perf_event, event_entry);
1963
1964 while (&event->event_entry != &ctx->event_list &&
1965 &next_event->event_entry != &next_ctx->event_list) {
1966
1967 __perf_event_sync_stat(event, next_event);
1968
1969 event = list_next_entry(event, event_entry);
1970 next_event = list_next_entry(next_event, event_entry);
1971 }
1972}
1973
1974static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1975 struct task_struct *next)
1976{
1977 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1978 struct perf_event_context *next_ctx;
1979 struct perf_event_context *parent;
1980 struct perf_cpu_context *cpuctx;
1981 int do_switch = 1;
1982
1983 if (likely(!ctx))
1984 return;
1985
1986 cpuctx = __get_cpu_context(ctx);
1987 if (!cpuctx->task_ctx)
1988 return;
1989
1990 rcu_read_lock();
1991 parent = rcu_dereference(ctx->parent_ctx);
1992 next_ctx = next->perf_event_ctxp[ctxn];
1993 if (parent && next_ctx &&
1994 rcu_dereference(next_ctx->parent_ctx) == parent) {
1995 /*
1996 * Looks like the two contexts are clones, so we might be
1997 * able to optimize the context switch. We lock both
1998 * contexts and check that they are clones under the
1999 * lock (including re-checking that neither has been
2000 * uncloned in the meantime). It doesn't matter which
2001 * order we take the locks because no other cpu could
2002 * be trying to lock both of these tasks.
2003 */
2004 raw_spin_lock(&ctx->lock);
2005 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2006 if (context_equiv(ctx, next_ctx)) {
2007 /*
2008 * XXX do we need a memory barrier of sorts
2009 * wrt to rcu_dereference() of perf_event_ctxp
2010 */
2011 task->perf_event_ctxp[ctxn] = next_ctx;
2012 next->perf_event_ctxp[ctxn] = ctx;
2013 ctx->task = next;
2014 next_ctx->task = task;
2015 do_switch = 0;
2016
2017 perf_event_sync_stat(ctx, next_ctx);
2018 }
2019 raw_spin_unlock(&next_ctx->lock);
2020 raw_spin_unlock(&ctx->lock);
2021 }
2022 rcu_read_unlock();
2023
2024 if (do_switch) {
2025 raw_spin_lock(&ctx->lock);
2026 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2027 cpuctx->task_ctx = NULL;
2028 raw_spin_unlock(&ctx->lock);
2029 }
2030}
2031
2032#define for_each_task_context_nr(ctxn) \
2033 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2034
2035/*
2036 * Called from scheduler to remove the events of the current task,
2037 * with interrupts disabled.
2038 *
2039 * We stop each event and update the event value in event->count.
2040 *
2041 * This does not protect us against NMI, but disable()
2042 * sets the disabled bit in the control field of event _before_
2043 * accessing the event control register. If a NMI hits, then it will
2044 * not restart the event.
2045 */
2046void __perf_event_task_sched_out(struct task_struct *task,
2047 struct task_struct *next)
2048{
2049 int ctxn;
2050
2051 for_each_task_context_nr(ctxn)
2052 perf_event_context_sched_out(task, ctxn, next);
2053
2054 /*
2055 * if cgroup events exist on this CPU, then we need
2056 * to check if we have to switch out PMU state.
2057 * cgroup event are system-wide mode only
2058 */
2059 if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2060 perf_cgroup_sched_out(task, next);
2061}
2062
2063static void task_ctx_sched_out(struct perf_event_context *ctx)
2064{
2065 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2066
2067 if (!cpuctx->task_ctx)
2068 return;
2069
2070 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2071 return;
2072
2073 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2074 cpuctx->task_ctx = NULL;
2075}
2076
2077/*
2078 * Called with IRQs disabled
2079 */
2080static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2081 enum event_type_t event_type)
2082{
2083 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2084}
2085
2086static void
2087ctx_pinned_sched_in(struct perf_event_context *ctx,
2088 struct perf_cpu_context *cpuctx)
2089{
2090 struct perf_event *event;
2091
2092 list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2093 if (event->state <= PERF_EVENT_STATE_OFF)
2094 continue;
2095 if (!event_filter_match(event))
2096 continue;
2097
2098 /* may need to reset tstamp_enabled */
2099 if (is_cgroup_event(event))
2100 perf_cgroup_mark_enabled(event, ctx);
2101
2102 if (group_can_go_on(event, cpuctx, 1))
2103 group_sched_in(event, cpuctx, ctx);
2104
2105 /*
2106 * If this pinned group hasn't been scheduled,
2107 * put it in error state.
2108 */
2109 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2110 update_group_times(event);
2111 event->state = PERF_EVENT_STATE_ERROR;
2112 }
2113 }
2114}
2115
2116static void
2117ctx_flexible_sched_in(struct perf_event_context *ctx,
2118 struct perf_cpu_context *cpuctx)
2119{
2120 struct perf_event *event;
2121 int can_add_hw = 1;
2122
2123 list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2124 /* Ignore events in OFF or ERROR state */
2125 if (event->state <= PERF_EVENT_STATE_OFF)
2126 continue;
2127 /*
2128 * Listen to the 'cpu' scheduling filter constraint
2129 * of events:
2130 */
2131 if (!event_filter_match(event))
2132 continue;
2133
2134 /* may need to reset tstamp_enabled */
2135 if (is_cgroup_event(event))
2136 perf_cgroup_mark_enabled(event, ctx);
2137
2138 if (group_can_go_on(event, cpuctx, can_add_hw)) {
2139 if (group_sched_in(event, cpuctx, ctx))
2140 can_add_hw = 0;
2141 }
2142 }
2143}
2144
2145static void
2146ctx_sched_in(struct perf_event_context *ctx,
2147 struct perf_cpu_context *cpuctx,
2148 enum event_type_t event_type,
2149 struct task_struct *task)
2150{
2151 u64 now;
2152 int is_active = ctx->is_active;
2153
2154 ctx->is_active |= event_type;
2155 if (likely(!ctx->nr_events))
2156 return;
2157
2158 now = perf_clock();
2159 ctx->timestamp = now;
2160 perf_cgroup_set_timestamp(task, ctx);
2161 /*
2162 * First go through the list and put on any pinned groups
2163 * in order to give them the best chance of going on.
2164 */
2165 if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2166 ctx_pinned_sched_in(ctx, cpuctx);
2167
2168 /* Then walk through the lower prio flexible groups */
2169 if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2170 ctx_flexible_sched_in(ctx, cpuctx);
2171}
2172
2173static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2174 enum event_type_t event_type,
2175 struct task_struct *task)
2176{
2177 struct perf_event_context *ctx = &cpuctx->ctx;
2178
2179 ctx_sched_in(ctx, cpuctx, event_type, task);
2180}
2181
2182static void perf_event_context_sched_in(struct perf_event_context *ctx,
2183 struct task_struct *task)
2184{
2185 struct perf_cpu_context *cpuctx;
2186
2187 cpuctx = __get_cpu_context(ctx);
2188 if (cpuctx->task_ctx == ctx)
2189 return;
2190
2191 perf_ctx_lock(cpuctx, ctx);
2192 perf_pmu_disable(ctx->pmu);
2193 /*
2194 * We want to keep the following priority order:
2195 * cpu pinned (that don't need to move), task pinned,
2196 * cpu flexible, task flexible.
2197 */
2198 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2199
2200 if (ctx->nr_events)
2201 cpuctx->task_ctx = ctx;
2202
2203 perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2204
2205 perf_pmu_enable(ctx->pmu);
2206 perf_ctx_unlock(cpuctx, ctx);
2207
2208 /*
2209 * Since these rotations are per-cpu, we need to ensure the
2210 * cpu-context we got scheduled on is actually rotating.
2211 */
2212 perf_pmu_rotate_start(ctx->pmu);
2213}
2214
2215/*
2216 * When sampling the branck stack in system-wide, it may be necessary
2217 * to flush the stack on context switch. This happens when the branch
2218 * stack does not tag its entries with the pid of the current task.
2219 * Otherwise it becomes impossible to associate a branch entry with a
2220 * task. This ambiguity is more likely to appear when the branch stack
2221 * supports priv level filtering and the user sets it to monitor only
2222 * at the user level (which could be a useful measurement in system-wide
2223 * mode). In that case, the risk is high of having a branch stack with
2224 * branch from multiple tasks. Flushing may mean dropping the existing
2225 * entries or stashing them somewhere in the PMU specific code layer.
2226 *
2227 * This function provides the context switch callback to the lower code
2228 * layer. It is invoked ONLY when there is at least one system-wide context
2229 * with at least one active event using taken branch sampling.
2230 */
2231static void perf_branch_stack_sched_in(struct task_struct *prev,
2232 struct task_struct *task)
2233{
2234 struct perf_cpu_context *cpuctx;
2235 struct pmu *pmu;
2236 unsigned long flags;
2237
2238 /* no need to flush branch stack if not changing task */
2239 if (prev == task)
2240 return;
2241
2242 local_irq_save(flags);
2243
2244 rcu_read_lock();
2245
2246 list_for_each_entry_rcu(pmu, &pmus, entry) {
2247 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2248
2249 /*
2250 * check if the context has at least one
2251 * event using PERF_SAMPLE_BRANCH_STACK
2252 */
2253 if (cpuctx->ctx.nr_branch_stack > 0
2254 && pmu->flush_branch_stack) {
2255
2256 pmu = cpuctx->ctx.pmu;
2257
2258 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2259
2260 perf_pmu_disable(pmu);
2261
2262 pmu->flush_branch_stack();
2263
2264 perf_pmu_enable(pmu);
2265
2266 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2267 }
2268 }
2269
2270 rcu_read_unlock();
2271
2272 local_irq_restore(flags);
2273}
2274
2275/*
2276 * Called from scheduler to add the events of the current task
2277 * with interrupts disabled.
2278 *
2279 * We restore the event value and then enable it.
2280 *
2281 * This does not protect us against NMI, but enable()
2282 * sets the enabled bit in the control field of event _before_
2283 * accessing the event control register. If a NMI hits, then it will
2284 * keep the event running.
2285 */
2286void __perf_event_task_sched_in(struct task_struct *prev,
2287 struct task_struct *task)
2288{
2289 struct perf_event_context *ctx;
2290 int ctxn;
2291
2292 for_each_task_context_nr(ctxn) {
2293 ctx = task->perf_event_ctxp[ctxn];
2294 if (likely(!ctx))
2295 continue;
2296
2297 perf_event_context_sched_in(ctx, task);
2298 }
2299 /*
2300 * if cgroup events exist on this CPU, then we need
2301 * to check if we have to switch in PMU state.
2302 * cgroup event are system-wide mode only
2303 */
2304 if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2305 perf_cgroup_sched_in(prev, task);
2306
2307 /* check for system-wide branch_stack events */
2308 if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
2309 perf_branch_stack_sched_in(prev, task);
2310}
2311
2312static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2313{
2314 u64 frequency = event->attr.sample_freq;
2315 u64 sec = NSEC_PER_SEC;
2316 u64 divisor, dividend;
2317
2318 int count_fls, nsec_fls, frequency_fls, sec_fls;
2319
2320 count_fls = fls64(count);
2321 nsec_fls = fls64(nsec);
2322 frequency_fls = fls64(frequency);
2323 sec_fls = 30;
2324
2325 /*
2326 * We got @count in @nsec, with a target of sample_freq HZ
2327 * the target period becomes:
2328 *
2329 * @count * 10^9
2330 * period = -------------------
2331 * @nsec * sample_freq
2332 *
2333 */
2334
2335 /*
2336 * Reduce accuracy by one bit such that @a and @b converge
2337 * to a similar magnitude.
2338 */
2339#define REDUCE_FLS(a, b) \
2340do { \
2341 if (a##_fls > b##_fls) { \
2342 a >>= 1; \
2343 a##_fls--; \
2344 } else { \
2345 b >>= 1; \
2346 b##_fls--; \
2347 } \
2348} while (0)
2349
2350 /*
2351 * Reduce accuracy until either term fits in a u64, then proceed with
2352 * the other, so that finally we can do a u64/u64 division.
2353 */
2354 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2355 REDUCE_FLS(nsec, frequency);
2356 REDUCE_FLS(sec, count);
2357 }
2358
2359 if (count_fls + sec_fls > 64) {
2360 divisor = nsec * frequency;
2361
2362 while (count_fls + sec_fls > 64) {
2363 REDUCE_FLS(count, sec);
2364 divisor >>= 1;
2365 }
2366
2367 dividend = count * sec;
2368 } else {
2369 dividend = count * sec;
2370
2371 while (nsec_fls + frequency_fls > 64) {
2372 REDUCE_FLS(nsec, frequency);
2373 dividend >>= 1;
2374 }
2375
2376 divisor = nsec * frequency;
2377 }
2378
2379 if (!divisor)
2380 return dividend;
2381
2382 return div64_u64(dividend, divisor);
2383}
2384
2385static DEFINE_PER_CPU(int, perf_throttled_count);
2386static DEFINE_PER_CPU(u64, perf_throttled_seq);
2387
2388static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2389{
2390 struct hw_perf_event *hwc = &event->hw;
2391 s64 period, sample_period;
2392 s64 delta;
2393
2394 period = perf_calculate_period(event, nsec, count);
2395
2396 delta = (s64)(period - hwc->sample_period);
2397 delta = (delta + 7) / 8; /* low pass filter */
2398
2399 sample_period = hwc->sample_period + delta;
2400
2401 if (!sample_period)
2402 sample_period = 1;
2403
2404 hwc->sample_period = sample_period;
2405
2406 if (local64_read(&hwc->period_left) > 8*sample_period) {
2407 if (disable)
2408 event->pmu->stop(event, PERF_EF_UPDATE);
2409
2410 local64_set(&hwc->period_left, 0);
2411
2412 if (disable)
2413 event->pmu->start(event, PERF_EF_RELOAD);
2414 }
2415}
2416
2417/*
2418 * combine freq adjustment with unthrottling to avoid two passes over the
2419 * events. At the same time, make sure, having freq events does not change
2420 * the rate of unthrottling as that would introduce bias.
2421 */
2422static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2423 int needs_unthr)
2424{
2425 struct perf_event *event;
2426 struct hw_perf_event *hwc;
2427 u64 now, period = TICK_NSEC;
2428 s64 delta;
2429
2430 /*
2431 * only need to iterate over all events iff:
2432 * - context have events in frequency mode (needs freq adjust)
2433 * - there are events to unthrottle on this cpu
2434 */
2435 if (!(ctx->nr_freq || needs_unthr))
2436 return;
2437
2438 raw_spin_lock(&ctx->lock);
2439 perf_pmu_disable(ctx->pmu);
2440
2441 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2442 if (event->state != PERF_EVENT_STATE_ACTIVE)
2443 continue;
2444
2445 if (!event_filter_match(event))
2446 continue;
2447
2448 hwc = &event->hw;
2449
2450 if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
2451 hwc->interrupts = 0;
2452 perf_log_throttle(event, 1);
2453 event->pmu->start(event, 0);
2454 }
2455
2456 if (!event->attr.freq || !event->attr.sample_freq)
2457 continue;
2458
2459 /*
2460 * stop the event and update event->count
2461 */
2462 event->pmu->stop(event, PERF_EF_UPDATE);
2463
2464 now = local64_read(&event->count);
2465 delta = now - hwc->freq_count_stamp;
2466 hwc->freq_count_stamp = now;
2467
2468 /*
2469 * restart the event
2470 * reload only if value has changed
2471 * we have stopped the event so tell that
2472 * to perf_adjust_period() to avoid stopping it
2473 * twice.
2474 */
2475 if (delta > 0)
2476 perf_adjust_period(event, period, delta, false);
2477
2478 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2479 }
2480
2481 perf_pmu_enable(ctx->pmu);
2482 raw_spin_unlock(&ctx->lock);
2483}
2484
2485/*
2486 * Round-robin a context's events:
2487 */
2488static void rotate_ctx(struct perf_event_context *ctx)
2489{
2490 /*
2491 * Rotate the first entry last of non-pinned groups. Rotation might be
2492 * disabled by the inheritance code.
2493 */
2494 if (!ctx->rotate_disable)
2495 list_rotate_left(&ctx->flexible_groups);
2496}
2497
2498/*
2499 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2500 * because they're strictly cpu affine and rotate_start is called with IRQs
2501 * disabled, while rotate_context is called from IRQ context.
2502 */
2503static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2504{
2505 struct perf_event_context *ctx = NULL;
2506 int rotate = 0, remove = 1;
2507
2508 if (cpuctx->ctx.nr_events) {
2509 remove = 0;
2510 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2511 rotate = 1;
2512 }
2513
2514 ctx = cpuctx->task_ctx;
2515 if (ctx && ctx->nr_events) {
2516 remove = 0;
2517 if (ctx->nr_events != ctx->nr_active)
2518 rotate = 1;
2519 }
2520
2521 if (!rotate)
2522 goto done;
2523
2524 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2525 perf_pmu_disable(cpuctx->ctx.pmu);
2526
2527 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2528 if (ctx)
2529 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2530
2531 rotate_ctx(&cpuctx->ctx);
2532 if (ctx)
2533 rotate_ctx(ctx);
2534
2535 perf_event_sched_in(cpuctx, ctx, current);
2536
2537 perf_pmu_enable(cpuctx->ctx.pmu);
2538 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2539done:
2540 if (remove)
2541 list_del_init(&cpuctx->rotation_list);
2542}
2543
2544void perf_event_task_tick(void)
2545{
2546 struct list_head *head = &__get_cpu_var(rotation_list);
2547 struct perf_cpu_context *cpuctx, *tmp;
2548 struct perf_event_context *ctx;
2549 int throttled;
2550
2551 WARN_ON(!irqs_disabled());
2552
2553 __this_cpu_inc(perf_throttled_seq);
2554 throttled = __this_cpu_xchg(perf_throttled_count, 0);
2555
2556 list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2557 ctx = &cpuctx->ctx;
2558 perf_adjust_freq_unthr_context(ctx, throttled);
2559
2560 ctx = cpuctx->task_ctx;
2561 if (ctx)
2562 perf_adjust_freq_unthr_context(ctx, throttled);
2563
2564 if (cpuctx->jiffies_interval == 1 ||
2565 !(jiffies % cpuctx->jiffies_interval))
2566 perf_rotate_context(cpuctx);
2567 }
2568}
2569
2570static int event_enable_on_exec(struct perf_event *event,
2571 struct perf_event_context *ctx)
2572{
2573 if (!event->attr.enable_on_exec)
2574 return 0;
2575
2576 event->attr.enable_on_exec = 0;
2577 if (event->state >= PERF_EVENT_STATE_INACTIVE)
2578 return 0;
2579
2580 __perf_event_mark_enabled(event);
2581
2582 return 1;
2583}
2584
2585/*
2586 * Enable all of a task's events that have been marked enable-on-exec.
2587 * This expects task == current.
2588 */
2589static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2590{
2591 struct perf_event *event;
2592 unsigned long flags;
2593 int enabled = 0;
2594 int ret;
2595
2596 local_irq_save(flags);
2597 if (!ctx || !ctx->nr_events)
2598 goto out;
2599
2600 /*
2601 * We must ctxsw out cgroup events to avoid conflict
2602 * when invoking perf_task_event_sched_in() later on
2603 * in this function. Otherwise we end up trying to
2604 * ctxswin cgroup events which are already scheduled
2605 * in.
2606 */
2607 perf_cgroup_sched_out(current, NULL);
2608
2609 raw_spin_lock(&ctx->lock);
2610 task_ctx_sched_out(ctx);
2611
2612 list_for_each_entry(event, &ctx->event_list, event_entry) {
2613 ret = event_enable_on_exec(event, ctx);
2614 if (ret)
2615 enabled = 1;
2616 }
2617
2618 /*
2619 * Unclone this context if we enabled any event.
2620 */
2621 if (enabled)
2622 unclone_ctx(ctx);
2623
2624 raw_spin_unlock(&ctx->lock);
2625
2626 /*
2627 * Also calls ctxswin for cgroup events, if any:
2628 */
2629 perf_event_context_sched_in(ctx, ctx->task);
2630out:
2631 local_irq_restore(flags);
2632}
2633
2634/*
2635 * Cross CPU call to read the hardware event
2636 */
2637static void __perf_event_read(void *info)
2638{
2639 struct perf_event *event = info;
2640 struct perf_event_context *ctx = event->ctx;
2641 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2642
2643 /*
2644 * If this is a task context, we need to check whether it is
2645 * the current task context of this cpu. If not it has been
2646 * scheduled out before the smp call arrived. In that case
2647 * event->count would have been updated to a recent sample
2648 * when the event was scheduled out.
2649 */
2650 if (ctx->task && cpuctx->task_ctx != ctx)
2651 return;
2652
2653 raw_spin_lock(&ctx->lock);
2654 if (ctx->is_active) {
2655 update_context_time(ctx);
2656 update_cgrp_time_from_event(event);
2657 }
2658 update_event_times(event);
2659 if (event->state == PERF_EVENT_STATE_ACTIVE)
2660 event->pmu->read(event);
2661 raw_spin_unlock(&ctx->lock);
2662}
2663
2664static inline u64 perf_event_count(struct perf_event *event)
2665{
2666 return local64_read(&event->count) + atomic64_read(&event->child_count);
2667}
2668
2669static u64 perf_event_read(struct perf_event *event)
2670{
2671 /*
2672 * If event is enabled and currently active on a CPU, update the
2673 * value in the event structure:
2674 */
2675 if (event->state == PERF_EVENT_STATE_ACTIVE) {
2676 smp_call_function_single(event->oncpu,
2677 __perf_event_read, event, 1);
2678 } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
2679 struct perf_event_context *ctx = event->ctx;
2680 unsigned long flags;
2681
2682 raw_spin_lock_irqsave(&ctx->lock, flags);
2683 /*
2684 * may read while context is not active
2685 * (e.g., thread is blocked), in that case
2686 * we cannot update context time
2687 */
2688 if (ctx->is_active) {
2689 update_context_time(ctx);
2690 update_cgrp_time_from_event(event);
2691 }
2692 update_event_times(event);
2693 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2694 }
2695
2696 return perf_event_count(event);
2697}
2698
2699/*
2700 * Initialize the perf_event context in a task_struct:
2701 */
2702static void __perf_event_init_context(struct perf_event_context *ctx)
2703{
2704 raw_spin_lock_init(&ctx->lock);
2705 mutex_init(&ctx->mutex);
2706 INIT_LIST_HEAD(&ctx->pinned_groups);
2707 INIT_LIST_HEAD(&ctx->flexible_groups);
2708 INIT_LIST_HEAD(&ctx->event_list);
2709 atomic_set(&ctx->refcount, 1);
2710}
2711
2712static struct perf_event_context *
2713alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2714{
2715 struct perf_event_context *ctx;
2716
2717 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2718 if (!ctx)
2719 return NULL;
2720
2721 __perf_event_init_context(ctx);
2722 if (task) {
2723 ctx->task = task;
2724 get_task_struct(task);
2725 }
2726 ctx->pmu = pmu;
2727
2728 return ctx;
2729}
2730
2731static struct task_struct *
2732find_lively_task_by_vpid(pid_t vpid)
2733{
2734 struct task_struct *task;
2735 int err;
2736
2737 rcu_read_lock();
2738 if (!vpid)
2739 task = current;
2740 else
2741 task = find_task_by_vpid(vpid);
2742 if (task)
2743 get_task_struct(task);
2744 rcu_read_unlock();
2745
2746 if (!task)
2747 return ERR_PTR(-ESRCH);
2748
2749 /* Reuse ptrace permission checks for now. */
2750 err = -EACCES;
2751 if (!ptrace_may_access(task, PTRACE_MODE_READ))
2752 goto errout;
2753
2754 return task;
2755errout:
2756 put_task_struct(task);
2757 return ERR_PTR(err);
2758
2759}
2760
2761/*
2762 * Returns a matching context with refcount and pincount.
2763 */
2764static struct perf_event_context *
2765find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2766{
2767 struct perf_event_context *ctx;
2768 struct perf_cpu_context *cpuctx;
2769 unsigned long flags;
2770 int ctxn, err;
2771
2772 if (!task) {
2773 /* Must be root to operate on a CPU event: */
2774 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2775 return ERR_PTR(-EACCES);
2776
2777 /*
2778 * We could be clever and allow to attach a event to an
2779 * offline CPU and activate it when the CPU comes up, but
2780 * that's for later.
2781 */
2782 if (!cpu_online(cpu))
2783 return ERR_PTR(-ENODEV);
2784
2785 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2786 ctx = &cpuctx->ctx;
2787 get_ctx(ctx);
2788 ++ctx->pin_count;
2789
2790 return ctx;
2791 }
2792
2793 err = -EINVAL;
2794 ctxn = pmu->task_ctx_nr;
2795 if (ctxn < 0)
2796 goto errout;
2797
2798retry:
2799 ctx = perf_lock_task_context(task, ctxn, &flags);
2800 if (ctx) {
2801 unclone_ctx(ctx);
2802 ++ctx->pin_count;
2803 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2804 } else {
2805 ctx = alloc_perf_context(pmu, task);
2806 err = -ENOMEM;
2807 if (!ctx)
2808 goto errout;
2809
2810 err = 0;
2811 mutex_lock(&task->perf_event_mutex);
2812 /*
2813 * If it has already passed perf_event_exit_task().
2814 * we must see PF_EXITING, it takes this mutex too.
2815 */
2816 if (task->flags & PF_EXITING)
2817 err = -ESRCH;
2818 else if (task->perf_event_ctxp[ctxn])
2819 err = -EAGAIN;
2820 else {
2821 get_ctx(ctx);
2822 ++ctx->pin_count;
2823 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2824 }
2825 mutex_unlock(&task->perf_event_mutex);
2826
2827 if (unlikely(err)) {
2828 put_ctx(ctx);
2829
2830 if (err == -EAGAIN)
2831 goto retry;
2832 goto errout;
2833 }
2834 }
2835
2836 return ctx;
2837
2838errout:
2839 return ERR_PTR(err);
2840}
2841
2842static void perf_event_free_filter(struct perf_event *event);
2843
2844static void free_event_rcu(struct rcu_head *head)
2845{
2846 struct perf_event *event;
2847
2848 event = container_of(head, struct perf_event, rcu_head);
2849 if (event->ns)
2850 put_pid_ns(event->ns);
2851 perf_event_free_filter(event);
2852 kfree(event);
2853}
2854
2855static void ring_buffer_put(struct ring_buffer *rb);
2856
2857static void free_event(struct perf_event *event)
2858{
2859 irq_work_sync(&event->pending);
2860
2861 if (!event->parent) {
2862 if (event->attach_state & PERF_ATTACH_TASK)
2863 static_key_slow_dec_deferred(&perf_sched_events);
2864 if (event->attr.mmap || event->attr.mmap_data)
2865 atomic_dec(&nr_mmap_events);
2866 if (event->attr.comm)
2867 atomic_dec(&nr_comm_events);
2868 if (event->attr.task)
2869 atomic_dec(&nr_task_events);
2870 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2871 put_callchain_buffers();
2872 if (is_cgroup_event(event)) {
2873 atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
2874 static_key_slow_dec_deferred(&perf_sched_events);
2875 }
2876
2877 if (has_branch_stack(event)) {
2878 static_key_slow_dec_deferred(&perf_sched_events);
2879 /* is system-wide event */
2880 if (!(event->attach_state & PERF_ATTACH_TASK))
2881 atomic_dec(&per_cpu(perf_branch_stack_events,
2882 event->cpu));
2883 }
2884 }
2885
2886 if (event->rb) {
2887 ring_buffer_put(event->rb);
2888 event->rb = NULL;
2889 }
2890
2891 if (is_cgroup_event(event))
2892 perf_detach_cgroup(event);
2893
2894 if (event->destroy)
2895 event->destroy(event);
2896
2897 if (event->ctx)
2898 put_ctx(event->ctx);
2899
2900 call_rcu(&event->rcu_head, free_event_rcu);
2901}
2902
2903int perf_event_release_kernel(struct perf_event *event)
2904{
2905 struct perf_event_context *ctx = event->ctx;
2906
2907 WARN_ON_ONCE(ctx->parent_ctx);
2908 /*
2909 * There are two ways this annotation is useful:
2910 *
2911 * 1) there is a lock recursion from perf_event_exit_task
2912 * see the comment there.
2913 *
2914 * 2) there is a lock-inversion with mmap_sem through
2915 * perf_event_read_group(), which takes faults while
2916 * holding ctx->mutex, however this is called after
2917 * the last filedesc died, so there is no possibility
2918 * to trigger the AB-BA case.
2919 */
2920 mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2921 raw_spin_lock_irq(&ctx->lock);
2922 perf_group_detach(event);
2923 raw_spin_unlock_irq(&ctx->lock);
2924 perf_remove_from_context(event);
2925 mutex_unlock(&ctx->mutex);
2926
2927 free_event(event);
2928
2929 return 0;
2930}
2931EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2932
2933/*
2934 * Called when the last reference to the file is gone.
2935 */
2936static void put_event(struct perf_event *event)
2937{
2938 struct task_struct *owner;
2939
2940 if (!atomic_long_dec_and_test(&event->refcount))
2941 return;
2942
2943 rcu_read_lock();
2944 owner = ACCESS_ONCE(event->owner);
2945 /*
2946 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
2947 * !owner it means the list deletion is complete and we can indeed
2948 * free this event, otherwise we need to serialize on
2949 * owner->perf_event_mutex.
2950 */
2951 smp_read_barrier_depends();
2952 if (owner) {
2953 /*
2954 * Since delayed_put_task_struct() also drops the last
2955 * task reference we can safely take a new reference
2956 * while holding the rcu_read_lock().
2957 */
2958 get_task_struct(owner);
2959 }
2960 rcu_read_unlock();
2961
2962 if (owner) {
2963 mutex_lock(&owner->perf_event_mutex);
2964 /*
2965 * We have to re-check the event->owner field, if it is cleared
2966 * we raced with perf_event_exit_task(), acquiring the mutex
2967 * ensured they're done, and we can proceed with freeing the
2968 * event.
2969 */
2970 if (event->owner)
2971 list_del_init(&event->owner_entry);
2972 mutex_unlock(&owner->perf_event_mutex);
2973 put_task_struct(owner);
2974 }
2975
2976 perf_event_release_kernel(event);
2977}
2978
2979static int perf_release(struct inode *inode, struct file *file)
2980{
2981 put_event(file->private_data);
2982 return 0;
2983}
2984
2985u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2986{
2987 struct perf_event *child;
2988 u64 total = 0;
2989
2990 *enabled = 0;
2991 *running = 0;
2992
2993 mutex_lock(&event->child_mutex);
2994 total += perf_event_read(event);
2995 *enabled += event->total_time_enabled +
2996 atomic64_read(&event->child_total_time_enabled);
2997 *running += event->total_time_running +
2998 atomic64_read(&event->child_total_time_running);
2999
3000 list_for_each_entry(child, &event->child_list, child_list) {
3001 total += perf_event_read(child);
3002 *enabled += child->total_time_enabled;
3003 *running += child->total_time_running;
3004 }
3005 mutex_unlock(&event->child_mutex);
3006
3007 return total;
3008}
3009EXPORT_SYMBOL_GPL(perf_event_read_value);
3010
3011static int perf_event_read_group(struct perf_event *event,
3012 u64 read_format, char __user *buf)
3013{
3014 struct perf_event *leader = event->group_leader, *sub;
3015 int n = 0, size = 0, ret = -EFAULT;
3016 struct perf_event_context *ctx = leader->ctx;
3017 u64 values[5];
3018 u64 count, enabled, running;
3019
3020 mutex_lock(&ctx->mutex);
3021 count = perf_event_read_value(leader, &enabled, &running);
3022
3023 values[n++] = 1 + leader->nr_siblings;
3024 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3025 values[n++] = enabled;
3026 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3027 values[n++] = running;
3028 values[n++] = count;
3029 if (read_format & PERF_FORMAT_ID)
3030 values[n++] = primary_event_id(leader);
3031
3032 size = n * sizeof(u64);
3033
3034 if (copy_to_user(buf, values, size))
3035 goto unlock;
3036
3037 ret = size;
3038
3039 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3040 n = 0;
3041
3042 values[n++] = perf_event_read_value(sub, &enabled, &running);
3043 if (read_format & PERF_FORMAT_ID)
3044 values[n++] = primary_event_id(sub);
3045
3046 size = n * sizeof(u64);
3047
3048 if (copy_to_user(buf + ret, values, size)) {
3049 ret = -EFAULT;
3050 goto unlock;
3051 }
3052
3053 ret += size;
3054 }
3055unlock:
3056 mutex_unlock(&ctx->mutex);
3057
3058 return ret;
3059}
3060
3061static int perf_event_read_one(struct perf_event *event,
3062 u64 read_format, char __user *buf)
3063{
3064 u64 enabled, running;
3065 u64 values[4];
3066 int n = 0;
3067
3068 values[n++] = perf_event_read_value(event, &enabled, &running);
3069 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3070 values[n++] = enabled;
3071 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3072 values[n++] = running;
3073 if (read_format & PERF_FORMAT_ID)
3074 values[n++] = primary_event_id(event);
3075
3076 if (copy_to_user(buf, values, n * sizeof(u64)))
3077 return -EFAULT;
3078
3079 return n * sizeof(u64);
3080}
3081
3082/*
3083 * Read the performance event - simple non blocking version for now
3084 */
3085static ssize_t
3086perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3087{
3088 u64 read_format = event->attr.read_format;
3089 int ret;
3090
3091 /*
3092 * Return end-of-file for a read on a event that is in
3093 * error state (i.e. because it was pinned but it couldn't be
3094 * scheduled on to the CPU at some point).
3095 */
3096 if (event->state == PERF_EVENT_STATE_ERROR)
3097 return 0;
3098
3099 if (count < event->read_size)
3100 return -ENOSPC;
3101
3102 WARN_ON_ONCE(event->ctx->parent_ctx);
3103 if (read_format & PERF_FORMAT_GROUP)
3104 ret = perf_event_read_group(event, read_format, buf);
3105 else
3106 ret = perf_event_read_one(event, read_format, buf);
3107
3108 return ret;
3109}
3110
3111static ssize_t
3112perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3113{
3114 struct perf_event *event = file->private_data;
3115
3116 return perf_read_hw(event, buf, count);
3117}
3118
3119static unsigned int perf_poll(struct file *file, poll_table *wait)
3120{
3121 struct perf_event *event = file->private_data;
3122 struct ring_buffer *rb;
3123 unsigned int events = POLL_HUP;
3124
3125 /*
3126 * Race between perf_event_set_output() and perf_poll(): perf_poll()
3127 * grabs the rb reference but perf_event_set_output() overrides it.
3128 * Here is the timeline for two threads T1, T2:
3129 * t0: T1, rb = rcu_dereference(event->rb)
3130 * t1: T2, old_rb = event->rb
3131 * t2: T2, event->rb = new rb
3132 * t3: T2, ring_buffer_detach(old_rb)
3133 * t4: T1, ring_buffer_attach(rb1)
3134 * t5: T1, poll_wait(event->waitq)
3135 *
3136 * To avoid this problem, we grab mmap_mutex in perf_poll()
3137 * thereby ensuring that the assignment of the new ring buffer
3138 * and the detachment of the old buffer appear atomic to perf_poll()
3139 */
3140 mutex_lock(&event->mmap_mutex);
3141
3142 rcu_read_lock();
3143 rb = rcu_dereference(event->rb);
3144 if (rb) {
3145 ring_buffer_attach(event, rb);
3146 events = atomic_xchg(&rb->poll, 0);
3147 }
3148 rcu_read_unlock();
3149
3150 mutex_unlock(&event->mmap_mutex);
3151
3152 poll_wait(file, &event->waitq, wait);
3153
3154 return events;
3155}
3156
3157static void perf_event_reset(struct perf_event *event)
3158{
3159 (void)perf_event_read(event);
3160 local64_set(&event->count, 0);
3161 perf_event_update_userpage(event);
3162}
3163
3164/*
3165 * Holding the top-level event's child_mutex means that any
3166 * descendant process that has inherited this event will block
3167 * in sync_child_event if it goes to exit, thus satisfying the
3168 * task existence requirements of perf_event_enable/disable.
3169 */
3170static void perf_event_for_each_child(struct perf_event *event,
3171 void (*func)(struct perf_event *))
3172{
3173 struct perf_event *child;
3174
3175 WARN_ON_ONCE(event->ctx->parent_ctx);
3176 mutex_lock(&event->child_mutex);
3177 func(event);
3178 list_for_each_entry(child, &event->child_list, child_list)
3179 func(child);
3180 mutex_unlock(&event->child_mutex);
3181}
3182
3183static void perf_event_for_each(struct perf_event *event,
3184 void (*func)(struct perf_event *))
3185{
3186 struct perf_event_context *ctx = event->ctx;
3187 struct perf_event *sibling;
3188
3189 WARN_ON_ONCE(ctx->parent_ctx);
3190 mutex_lock(&ctx->mutex);
3191 event = event->group_leader;
3192
3193 perf_event_for_each_child(event, func);
3194 list_for_each_entry(sibling, &event->sibling_list, group_entry)
3195 perf_event_for_each_child(sibling, func);
3196 mutex_unlock(&ctx->mutex);
3197}
3198
3199static int perf_event_period(struct perf_event *event, u64 __user *arg)
3200{
3201 struct perf_event_context *ctx = event->ctx;
3202 int ret = 0;
3203 u64 value;
3204
3205 if (!is_sampling_event(event))
3206 return -EINVAL;
3207
3208 if (copy_from_user(&value, arg, sizeof(value)))
3209 return -EFAULT;
3210
3211 if (!value)
3212 return -EINVAL;
3213
3214 raw_spin_lock_irq(&ctx->lock);
3215 if (event->attr.freq) {
3216 if (value > sysctl_perf_event_sample_rate) {
3217 ret = -EINVAL;
3218 goto unlock;
3219 }
3220
3221 event->attr.sample_freq = value;
3222 } else {
3223 event->attr.sample_period = value;
3224 event->hw.sample_period = value;
3225 }
3226unlock:
3227 raw_spin_unlock_irq(&ctx->lock);
3228
3229 return ret;
3230}
3231
3232static const struct file_operations perf_fops;
3233
3234static struct file *perf_fget_light(int fd, int *fput_needed)
3235{
3236 struct file *file;
3237
3238 file = fget_light(fd, fput_needed);
3239 if (!file)
3240 return ERR_PTR(-EBADF);
3241
3242 if (file->f_op != &perf_fops) {
3243 fput_light(file, *fput_needed);
3244 *fput_needed = 0;
3245 return ERR_PTR(-EBADF);
3246 }
3247
3248 return file;
3249}
3250
3251static int perf_event_set_output(struct perf_event *event,
3252 struct perf_event *output_event);
3253static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3254
3255static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3256{
3257 struct perf_event *event = file->private_data;
3258 void (*func)(struct perf_event *);
3259 u32 flags = arg;
3260
3261 switch (cmd) {
3262 case PERF_EVENT_IOC_ENABLE:
3263 func = perf_event_enable;
3264 break;
3265 case PERF_EVENT_IOC_DISABLE:
3266 func = perf_event_disable;
3267 break;
3268 case PERF_EVENT_IOC_RESET:
3269 func = perf_event_reset;
3270 break;
3271
3272 case PERF_EVENT_IOC_REFRESH:
3273 return perf_event_refresh(event, arg);
3274
3275 case PERF_EVENT_IOC_PERIOD:
3276 return perf_event_period(event, (u64 __user *)arg);
3277
3278 case PERF_EVENT_IOC_SET_OUTPUT:
3279 {
3280 struct file *output_file = NULL;
3281 struct perf_event *output_event = NULL;
3282 int fput_needed = 0;
3283 int ret;
3284
3285 if (arg != -1) {
3286 output_file = perf_fget_light(arg, &fput_needed);
3287 if (IS_ERR(output_file))
3288 return PTR_ERR(output_file);
3289 output_event = output_file->private_data;
3290 }
3291
3292 ret = perf_event_set_output(event, output_event);
3293 if (output_event)
3294 fput_light(output_file, fput_needed);
3295
3296 return ret;
3297 }
3298
3299 case PERF_EVENT_IOC_SET_FILTER:
3300 return perf_event_set_filter(event, (void __user *)arg);
3301
3302 default:
3303 return -ENOTTY;
3304 }
3305
3306 if (flags & PERF_IOC_FLAG_GROUP)
3307 perf_event_for_each(event, func);
3308 else
3309 perf_event_for_each_child(event, func);
3310
3311 return 0;
3312}
3313
3314int perf_event_task_enable(void)
3315{
3316 struct perf_event *event;
3317
3318 mutex_lock(¤t->perf_event_mutex);
3319 list_for_each_entry(event, ¤t->perf_event_list, owner_entry)
3320 perf_event_for_each_child(event, perf_event_enable);
3321 mutex_unlock(¤t->perf_event_mutex);
3322
3323 return 0;
3324}
3325
3326int perf_event_task_disable(void)
3327{
3328 struct perf_event *event;
3329
3330 mutex_lock(¤t->perf_event_mutex);
3331 list_for_each_entry(event, ¤t->perf_event_list, owner_entry)
3332 perf_event_for_each_child(event, perf_event_disable);
3333 mutex_unlock(¤t->perf_event_mutex);
3334
3335 return 0;
3336}
3337
3338static int perf_event_index(struct perf_event *event)
3339{
3340 if (event->hw.state & PERF_HES_STOPPED)
3341 return 0;
3342
3343 if (event->state != PERF_EVENT_STATE_ACTIVE)
3344 return 0;
3345
3346 return event->pmu->event_idx(event);
3347}
3348
3349static void calc_timer_values(struct perf_event *event,
3350 u64 *now,
3351 u64 *enabled,
3352 u64 *running)
3353{
3354 u64 ctx_time;
3355
3356 *now = perf_clock();
3357 ctx_time = event->shadow_ctx_time + *now;
3358 *enabled = ctx_time - event->tstamp_enabled;
3359 *running = ctx_time - event->tstamp_running;
3360}
3361
3362void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
3363{
3364}
3365
3366/*
3367 * Callers need to ensure there can be no nesting of this function, otherwise
3368 * the seqlock logic goes bad. We can not serialize this because the arch
3369 * code calls this from NMI context.
3370 */
3371void perf_event_update_userpage(struct perf_event *event)
3372{
3373 struct perf_event_mmap_page *userpg;
3374 struct ring_buffer *rb;
3375 u64 enabled, running, now;
3376
3377 rcu_read_lock();
3378 /*
3379 * compute total_time_enabled, total_time_running
3380 * based on snapshot values taken when the event
3381 * was last scheduled in.
3382 *
3383 * we cannot simply called update_context_time()
3384 * because of locking issue as we can be called in
3385 * NMI context
3386 */
3387 calc_timer_values(event, &now, &enabled, &running);
3388 rb = rcu_dereference(event->rb);
3389 if (!rb)
3390 goto unlock;
3391
3392 userpg = rb->user_page;
3393
3394 /*
3395 * Disable preemption so as to not let the corresponding user-space
3396 * spin too long if we get preempted.
3397 */
3398 preempt_disable();
3399 ++userpg->lock;
3400 barrier();
3401 userpg->index = perf_event_index(event);
3402 userpg->offset = perf_event_count(event);
3403 if (userpg->index)
3404 userpg->offset -= local64_read(&event->hw.prev_count);
3405
3406 userpg->time_enabled = enabled +
3407 atomic64_read(&event->child_total_time_enabled);
3408
3409 userpg->time_running = running +
3410 atomic64_read(&event->child_total_time_running);
3411
3412 arch_perf_update_userpage(userpg, now);
3413
3414 barrier();
3415 ++userpg->lock;
3416 preempt_enable();
3417unlock:
3418 rcu_read_unlock();
3419}
3420
3421static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3422{
3423 struct perf_event *event = vma->vm_file->private_data;
3424 struct ring_buffer *rb;
3425 int ret = VM_FAULT_SIGBUS;
3426
3427 if (vmf->flags & FAULT_FLAG_MKWRITE) {
3428 if (vmf->pgoff == 0)
3429 ret = 0;
3430 return ret;
3431 }
3432
3433 rcu_read_lock();
3434 rb = rcu_dereference(event->rb);
3435 if (!rb)
3436 goto unlock;
3437
3438 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3439 goto unlock;
3440
3441 vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
3442 if (!vmf->page)
3443 goto unlock;
3444
3445 get_page(vmf->page);
3446 vmf->page->mapping = vma->vm_file->f_mapping;
3447 vmf->page->index = vmf->pgoff;
3448
3449 ret = 0;
3450unlock:
3451 rcu_read_unlock();
3452
3453 return ret;
3454}
3455
3456static void ring_buffer_attach(struct perf_event *event,
3457 struct ring_buffer *rb)
3458{
3459 unsigned long flags;
3460
3461 if (!list_empty(&event->rb_entry))
3462 return;
3463
3464 spin_lock_irqsave(&rb->event_lock, flags);
3465 if (!list_empty(&event->rb_entry))
3466 goto unlock;
3467
3468 list_add(&event->rb_entry, &rb->event_list);
3469unlock:
3470 spin_unlock_irqrestore(&rb->event_lock, flags);
3471}
3472
3473static void ring_buffer_detach(struct perf_event *event,
3474 struct ring_buffer *rb)
3475{
3476 unsigned long flags;
3477
3478 if (list_empty(&event->rb_entry))
3479 return;
3480
3481 spin_lock_irqsave(&rb->event_lock, flags);
3482 list_del_init(&event->rb_entry);
3483 wake_up_all(&event->waitq);
3484 spin_unlock_irqrestore(&rb->event_lock, flags);
3485}
3486
3487static void ring_buffer_wakeup(struct perf_event *event)
3488{
3489 struct ring_buffer *rb;
3490
3491 rcu_read_lock();
3492 rb = rcu_dereference(event->rb);
3493 if (!rb)
3494 goto unlock;
3495
3496 list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3497 wake_up_all(&event->waitq);
3498
3499unlock:
3500 rcu_read_unlock();
3501}
3502
3503static void rb_free_rcu(struct rcu_head *rcu_head)
3504{
3505 struct ring_buffer *rb;
3506
3507 rb = container_of(rcu_head, struct ring_buffer, rcu_head);
3508 rb_free(rb);
3509}
3510
3511static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3512{
3513 struct ring_buffer *rb;
3514
3515 rcu_read_lock();
3516 rb = rcu_dereference(event->rb);
3517 if (rb) {
3518 if (!atomic_inc_not_zero(&rb->refcount))
3519 rb = NULL;
3520 }
3521 rcu_read_unlock();
3522
3523 return rb;
3524}
3525
3526static void ring_buffer_put(struct ring_buffer *rb)
3527{
3528 struct perf_event *event, *n;
3529 unsigned long flags;
3530
3531 if (!atomic_dec_and_test(&rb->refcount))
3532 return;
3533
3534 spin_lock_irqsave(&rb->event_lock, flags);
3535 list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) {
3536 list_del_init(&event->rb_entry);
3537 wake_up_all(&event->waitq);
3538 }
3539 spin_unlock_irqrestore(&rb->event_lock, flags);
3540
3541 call_rcu(&rb->rcu_head, rb_free_rcu);
3542}
3543
3544static void perf_mmap_open(struct vm_area_struct *vma)
3545{
3546 struct perf_event *event = vma->vm_file->private_data;
3547
3548 atomic_inc(&event->mmap_count);
3549}
3550
3551static void perf_mmap_close(struct vm_area_struct *vma)
3552{
3553 struct perf_event *event = vma->vm_file->private_data;
3554
3555 if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3556 unsigned long size = perf_data_size(event->rb);
3557 struct user_struct *user = event->mmap_user;
3558 struct ring_buffer *rb = event->rb;
3559
3560 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3561 vma->vm_mm->pinned_vm -= event->mmap_locked;
3562 rcu_assign_pointer(event->rb, NULL);
3563 ring_buffer_detach(event, rb);
3564 mutex_unlock(&event->mmap_mutex);
3565
3566 ring_buffer_put(rb);
3567 free_uid(user);
3568 }
3569}
3570
3571static const struct vm_operations_struct perf_mmap_vmops = {
3572 .open = perf_mmap_open,
3573 .close = perf_mmap_close,
3574 .fault = perf_mmap_fault,
3575 .page_mkwrite = perf_mmap_fault,
3576};
3577
3578static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3579{
3580 struct perf_event *event = file->private_data;
3581 unsigned long user_locked, user_lock_limit;
3582 struct user_struct *user = current_user();
3583 unsigned long locked, lock_limit;
3584 struct ring_buffer *rb;
3585 unsigned long vma_size;
3586 unsigned long nr_pages;
3587 long user_extra, extra;
3588 int ret = 0, flags = 0;
3589
3590 /*
3591 * Don't allow mmap() of inherited per-task counters. This would
3592 * create a performance issue due to all children writing to the
3593 * same rb.
3594 */
3595 if (event->cpu == -1 && event->attr.inherit)
3596 return -EINVAL;
3597
3598 if (!(vma->vm_flags & VM_SHARED))
3599 return -EINVAL;
3600
3601 vma_size = vma->vm_end - vma->vm_start;
3602 nr_pages = (vma_size / PAGE_SIZE) - 1;
3603
3604 /*
3605 * If we have rb pages ensure they're a power-of-two number, so we
3606 * can do bitmasks instead of modulo.
3607 */
3608 if (nr_pages != 0 && !is_power_of_2(nr_pages))
3609 return -EINVAL;
3610
3611 if (vma_size != PAGE_SIZE * (1 + nr_pages))
3612 return -EINVAL;
3613
3614 if (vma->vm_pgoff != 0)
3615 return -EINVAL;
3616
3617 WARN_ON_ONCE(event->ctx->parent_ctx);
3618 mutex_lock(&event->mmap_mutex);
3619 if (event->rb) {
3620 if (event->rb->nr_pages == nr_pages)
3621 atomic_inc(&event->rb->refcount);
3622 else
3623 ret = -EINVAL;
3624 goto unlock;
3625 }
3626
3627 user_extra = nr_pages + 1;
3628 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3629
3630 /*
3631 * Increase the limit linearly with more CPUs:
3632 */
3633 user_lock_limit *= num_online_cpus();
3634
3635 user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3636
3637 extra = 0;
3638 if (user_locked > user_lock_limit)
3639 extra = user_locked - user_lock_limit;
3640
3641 lock_limit = rlimit(RLIMIT_MEMLOCK);
3642 lock_limit >>= PAGE_SHIFT;
3643 locked = vma->vm_mm->pinned_vm + extra;
3644
3645 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3646 !capable(CAP_IPC_LOCK)) {
3647 ret = -EPERM;
3648 goto unlock;
3649 }
3650
3651 WARN_ON(event->rb);
3652
3653 if (vma->vm_flags & VM_WRITE)
3654 flags |= RING_BUFFER_WRITABLE;
3655
3656 rb = rb_alloc(nr_pages,
3657 event->attr.watermark ? event->attr.wakeup_watermark : 0,
3658 event->cpu, flags);
3659
3660 if (!rb) {
3661 ret = -ENOMEM;
3662 goto unlock;
3663 }
3664 rcu_assign_pointer(event->rb, rb);
3665
3666 atomic_long_add(user_extra, &user->locked_vm);
3667 event->mmap_locked = extra;
3668 event->mmap_user = get_current_user();
3669 vma->vm_mm->pinned_vm += event->mmap_locked;
3670
3671 perf_event_update_userpage(event);
3672
3673unlock:
3674 if (!ret)
3675 atomic_inc(&event->mmap_count);
3676 mutex_unlock(&event->mmap_mutex);
3677
3678 vma->vm_flags |= VM_RESERVED;
3679 vma->vm_ops = &perf_mmap_vmops;
3680
3681 return ret;
3682}
3683
3684static int perf_fasync(int fd, struct file *filp, int on)
3685{
3686 struct inode *inode = filp->f_path.dentry->d_inode;
3687 struct perf_event *event = filp->private_data;
3688 int retval;
3689
3690 mutex_lock(&inode->i_mutex);
3691 retval = fasync_helper(fd, filp, on, &event->fasync);
3692 mutex_unlock(&inode->i_mutex);
3693
3694 if (retval < 0)
3695 return retval;
3696
3697 return 0;
3698}
3699
3700static const struct file_operations perf_fops = {
3701 .llseek = no_llseek,
3702 .release = perf_release,
3703 .read = perf_read,
3704 .poll = perf_poll,
3705 .unlocked_ioctl = perf_ioctl,
3706 .compat_ioctl = perf_ioctl,
3707 .mmap = perf_mmap,
3708 .fasync = perf_fasync,
3709};
3710
3711/*
3712 * Perf event wakeup
3713 *
3714 * If there's data, ensure we set the poll() state and publish everything
3715 * to user-space before waking everybody up.
3716 */
3717
3718void perf_event_wakeup(struct perf_event *event)
3719{
3720 ring_buffer_wakeup(event);
3721
3722 if (event->pending_kill) {
3723 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3724 event->pending_kill = 0;
3725 }
3726}
3727
3728static void perf_pending_event(struct irq_work *entry)
3729{
3730 struct perf_event *event = container_of(entry,
3731 struct perf_event, pending);
3732
3733 if (event->pending_disable) {
3734 event->pending_disable = 0;
3735 __perf_event_disable(event);
3736 }
3737
3738 if (event->pending_wakeup) {
3739 event->pending_wakeup = 0;
3740 perf_event_wakeup(event);
3741 }
3742}
3743
3744/*
3745 * We assume there is only KVM supporting the callbacks.
3746 * Later on, we might change it to a list if there is
3747 * another virtualization implementation supporting the callbacks.
3748 */
3749struct perf_guest_info_callbacks *perf_guest_cbs;
3750
3751int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3752{
3753 perf_guest_cbs = cbs;
3754 return 0;
3755}
3756EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3757
3758int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3759{
3760 perf_guest_cbs = NULL;
3761 return 0;
3762}
3763EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3764
3765static void __perf_event_header__init_id(struct perf_event_header *header,
3766 struct perf_sample_data *data,
3767 struct perf_event *event)
3768{
3769 u64 sample_type = event->attr.sample_type;
3770
3771 data->type = sample_type;
3772 header->size += event->id_header_size;
3773
3774 if (sample_type & PERF_SAMPLE_TID) {
3775 /* namespace issues */
3776 data->tid_entry.pid = perf_event_pid(event, current);
3777 data->tid_entry.tid = perf_event_tid(event, current);
3778 }
3779
3780 if (sample_type & PERF_SAMPLE_TIME)
3781 data->time = perf_clock();
3782
3783 if (sample_type & PERF_SAMPLE_ID)
3784 data->id = primary_event_id(event);
3785
3786 if (sample_type & PERF_SAMPLE_STREAM_ID)
3787 data->stream_id = event->id;
3788
3789 if (sample_type & PERF_SAMPLE_CPU) {
3790 data->cpu_entry.cpu = raw_smp_processor_id();
3791 data->cpu_entry.reserved = 0;
3792 }
3793}
3794
3795void perf_event_header__init_id(struct perf_event_header *header,
3796 struct perf_sample_data *data,
3797 struct perf_event *event)
3798{
3799 if (event->attr.sample_id_all)
3800 __perf_event_header__init_id(header, data, event);
3801}
3802
3803static void __perf_event__output_id_sample(struct perf_output_handle *handle,
3804 struct perf_sample_data *data)
3805{
3806 u64 sample_type = data->type;
3807
3808 if (sample_type & PERF_SAMPLE_TID)
3809 perf_output_put(handle, data->tid_entry);
3810
3811 if (sample_type & PERF_SAMPLE_TIME)
3812 perf_output_put(handle, data->time);
3813
3814 if (sample_type & PERF_SAMPLE_ID)
3815 perf_output_put(handle, data->id);
3816
3817 if (sample_type & PERF_SAMPLE_STREAM_ID)
3818 perf_output_put(handle, data->stream_id);
3819
3820 if (sample_type & PERF_SAMPLE_CPU)
3821 perf_output_put(handle, data->cpu_entry);
3822}
3823
3824void perf_event__output_id_sample(struct perf_event *event,
3825 struct perf_output_handle *handle,
3826 struct perf_sample_data *sample)
3827{
3828 if (event->attr.sample_id_all)
3829 __perf_event__output_id_sample(handle, sample);
3830}
3831
3832static void perf_output_read_one(struct perf_output_handle *handle,
3833 struct perf_event *event,
3834 u64 enabled, u64 running)
3835{
3836 u64 read_format = event->attr.read_format;
3837 u64 values[4];
3838 int n = 0;
3839
3840 values[n++] = perf_event_count(event);
3841 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3842 values[n++] = enabled +
3843 atomic64_read(&event->child_total_time_enabled);
3844 }
3845 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3846 values[n++] = running +
3847 atomic64_read(&event->child_total_time_running);
3848 }
3849 if (read_format & PERF_FORMAT_ID)
3850 values[n++] = primary_event_id(event);
3851
3852 __output_copy(handle, values, n * sizeof(u64));
3853}
3854
3855/*
3856 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3857 */
3858static void perf_output_read_group(struct perf_output_handle *handle,
3859 struct perf_event *event,
3860 u64 enabled, u64 running)
3861{
3862 struct perf_event *leader = event->group_leader, *sub;
3863 u64 read_format = event->attr.read_format;
3864 u64 values[5];
3865 int n = 0;
3866
3867 values[n++] = 1 + leader->nr_siblings;
3868
3869 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3870 values[n++] = enabled;
3871
3872 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3873 values[n++] = running;
3874
3875 if (leader != event)
3876 leader->pmu->read(leader);
3877
3878 values[n++] = perf_event_count(leader);
3879 if (read_format & PERF_FORMAT_ID)
3880 values[n++] = primary_event_id(leader);
3881
3882 __output_copy(handle, values, n * sizeof(u64));
3883
3884 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3885 n = 0;
3886
3887 if (sub != event)
3888 sub->pmu->read(sub);
3889
3890 values[n++] = perf_event_count(sub);
3891 if (read_format & PERF_FORMAT_ID)
3892 values[n++] = primary_event_id(sub);
3893
3894 __output_copy(handle, values, n * sizeof(u64));
3895 }
3896}
3897
3898#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
3899 PERF_FORMAT_TOTAL_TIME_RUNNING)
3900
3901static void perf_output_read(struct perf_output_handle *handle,
3902 struct perf_event *event)
3903{
3904 u64 enabled = 0, running = 0, now;
3905 u64 read_format = event->attr.read_format;
3906
3907 /*
3908 * compute total_time_enabled, total_time_running
3909 * based on snapshot values taken when the event
3910 * was last scheduled in.
3911 *
3912 * we cannot simply called update_context_time()
3913 * because of locking issue as we are called in
3914 * NMI context
3915 */
3916 if (read_format & PERF_FORMAT_TOTAL_TIMES)
3917 calc_timer_values(event, &now, &enabled, &running);
3918
3919 if (event->attr.read_format & PERF_FORMAT_GROUP)
3920 perf_output_read_group(handle, event, enabled, running);
3921 else
3922 perf_output_read_one(handle, event, enabled, running);
3923}
3924
3925void perf_output_sample(struct perf_output_handle *handle,
3926 struct perf_event_header *header,
3927 struct perf_sample_data *data,
3928 struct perf_event *event)
3929{
3930 u64 sample_type = data->type;
3931
3932 perf_output_put(handle, *header);
3933
3934 if (sample_type & PERF_SAMPLE_IP)
3935 perf_output_put(handle, data->ip);
3936
3937 if (sample_type & PERF_SAMPLE_TID)
3938 perf_output_put(handle, data->tid_entry);
3939
3940 if (sample_type & PERF_SAMPLE_TIME)
3941 perf_output_put(handle, data->time);
3942
3943 if (sample_type & PERF_SAMPLE_ADDR)
3944 perf_output_put(handle, data->addr);
3945
3946 if (sample_type & PERF_SAMPLE_ID)
3947 perf_output_put(handle, data->id);
3948
3949 if (sample_type & PERF_SAMPLE_STREAM_ID)
3950 perf_output_put(handle, data->stream_id);
3951
3952 if (sample_type & PERF_SAMPLE_CPU)
3953 perf_output_put(handle, data->cpu_entry);
3954
3955 if (sample_type & PERF_SAMPLE_PERIOD)
3956 perf_output_put(handle, data->period);
3957
3958 if (sample_type & PERF_SAMPLE_READ)
3959 perf_output_read(handle, event);
3960
3961 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3962 if (data->callchain) {
3963 int size = 1;
3964
3965 if (data->callchain)
3966 size += data->callchain->nr;
3967
3968 size *= sizeof(u64);
3969
3970 __output_copy(handle, data->callchain, size);
3971 } else {
3972 u64 nr = 0;
3973 perf_output_put(handle, nr);
3974 }
3975 }
3976
3977 if (sample_type & PERF_SAMPLE_RAW) {
3978 if (data->raw) {
3979 perf_output_put(handle, data->raw->size);
3980 __output_copy(handle, data->raw->data,
3981 data->raw->size);
3982 } else {
3983 struct {
3984 u32 size;
3985 u32 data;
3986 } raw = {
3987 .size = sizeof(u32),
3988 .data = 0,
3989 };
3990 perf_output_put(handle, raw);
3991 }
3992 }
3993
3994 if (!event->attr.watermark) {
3995 int wakeup_events = event->attr.wakeup_events;
3996
3997 if (wakeup_events) {
3998 struct ring_buffer *rb = handle->rb;
3999 int events = local_inc_return(&rb->events);
4000
4001 if (events >= wakeup_events) {
4002 local_sub(wakeup_events, &rb->events);
4003 local_inc(&rb->wakeup);
4004 }
4005 }
4006 }
4007
4008 if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4009 if (data->br_stack) {
4010 size_t size;
4011
4012 size = data->br_stack->nr
4013 * sizeof(struct perf_branch_entry);
4014
4015 perf_output_put(handle, data->br_stack->nr);
4016 perf_output_copy(handle, data->br_stack->entries, size);
4017 } else {
4018 /*
4019 * we always store at least the value of nr
4020 */
4021 u64 nr = 0;
4022 perf_output_put(handle, nr);
4023 }
4024 }
4025}
4026
4027void perf_prepare_sample(struct perf_event_header *header,
4028 struct perf_sample_data *data,
4029 struct perf_event *event,
4030 struct pt_regs *regs)
4031{
4032 u64 sample_type = event->attr.sample_type;
4033
4034 header->type = PERF_RECORD_SAMPLE;
4035 header->size = sizeof(*header) + event->header_size;
4036
4037 header->misc = 0;
4038 header->misc |= perf_misc_flags(regs);
4039
4040 __perf_event_header__init_id(header, data, event);
4041
4042 if (sample_type & PERF_SAMPLE_IP)
4043 data->ip = perf_instruction_pointer(regs);
4044
4045 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4046 int size = 1;
4047
4048 data->callchain = perf_callchain(regs);
4049
4050 if (data->callchain)
4051 size += data->callchain->nr;
4052
4053 header->size += size * sizeof(u64);
4054 }
4055
4056 if (sample_type & PERF_SAMPLE_RAW) {
4057 int size = sizeof(u32);
4058
4059 if (data->raw)
4060 size += data->raw->size;
4061 else
4062 size += sizeof(u32);
4063
4064 WARN_ON_ONCE(size & (sizeof(u64)-1));
4065 header->size += size;
4066 }
4067
4068 if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4069 int size = sizeof(u64); /* nr */
4070 if (data->br_stack) {
4071 size += data->br_stack->nr
4072 * sizeof(struct perf_branch_entry);
4073 }
4074 header->size += size;
4075 }
4076}
4077
4078static void perf_event_output(struct perf_event *event,
4079 struct perf_sample_data *data,
4080 struct pt_regs *regs)
4081{
4082 struct perf_output_handle handle;
4083 struct perf_event_header header;
4084
4085 /* protect the callchain buffers */
4086 rcu_read_lock();
4087
4088 perf_prepare_sample(&header, data, event, regs);
4089
4090 if (perf_output_begin(&handle, event, header.size))
4091 goto exit;
4092
4093 perf_output_sample(&handle, &header, data, event);
4094
4095 perf_output_end(&handle);
4096
4097exit:
4098 rcu_read_unlock();
4099}
4100
4101/*
4102 * read event_id
4103 */
4104
4105struct perf_read_event {
4106 struct perf_event_header header;
4107
4108 u32 pid;
4109 u32 tid;
4110};
4111
4112static void
4113perf_event_read_event(struct perf_event *event,
4114 struct task_struct *task)
4115{
4116 struct perf_output_handle handle;
4117 struct perf_sample_data sample;
4118 struct perf_read_event read_event = {
4119 .header = {
4120 .type = PERF_RECORD_READ,
4121 .misc = 0,
4122 .size = sizeof(read_event) + event->read_size,
4123 },
4124 .pid = perf_event_pid(event, task),
4125 .tid = perf_event_tid(event, task),
4126 };
4127 int ret;
4128
4129 perf_event_header__init_id(&read_event.header, &sample, event);
4130 ret = perf_output_begin(&handle, event, read_event.header.size);
4131 if (ret)
4132 return;
4133
4134 perf_output_put(&handle, read_event);
4135 perf_output_read(&handle, event);
4136 perf_event__output_id_sample(event, &handle, &sample);
4137
4138 perf_output_end(&handle);
4139}
4140
4141/*
4142 * task tracking -- fork/exit
4143 *
4144 * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
4145 */
4146
4147struct perf_task_event {
4148 struct task_struct *task;
4149 struct perf_event_context *task_ctx;
4150
4151 struct {
4152 struct perf_event_header header;
4153
4154 u32 pid;
4155 u32 ppid;
4156 u32 tid;
4157 u32 ptid;
4158 u64 time;
4159 } event_id;
4160};
4161
4162static void perf_event_task_output(struct perf_event *event,
4163 struct perf_task_event *task_event)
4164{
4165 struct perf_output_handle handle;
4166 struct perf_sample_data sample;
4167 struct task_struct *task = task_event->task;
4168 int ret, size = task_event->event_id.header.size;
4169
4170 perf_event_header__init_id(&task_event->event_id.header, &sample, event);
4171
4172 ret = perf_output_begin(&handle, event,
4173 task_event->event_id.header.size);
4174 if (ret)
4175 goto out;
4176
4177 task_event->event_id.pid = perf_event_pid(event, task);
4178 task_event->event_id.ppid = perf_event_pid(event, current);
4179
4180 task_event->event_id.tid = perf_event_tid(event, task);
4181 task_event->event_id.ptid = perf_event_tid(event, current);
4182
4183 perf_output_put(&handle, task_event->event_id);
4184
4185 perf_event__output_id_sample(event, &handle, &sample);
4186
4187 perf_output_end(&handle);
4188out:
4189 task_event->event_id.header.size = size;
4190}
4191
4192static int perf_event_task_match(struct perf_event *event)
4193{
4194 if (event->state < PERF_EVENT_STATE_INACTIVE)
4195 return 0;
4196
4197 if (!event_filter_match(event))
4198 return 0;
4199
4200 if (event->attr.comm || event->attr.mmap ||
4201 event->attr.mmap_data || event->attr.task)
4202 return 1;
4203
4204 return 0;
4205}
4206
4207static void perf_event_task_ctx(struct perf_event_context *ctx,
4208 struct perf_task_event *task_event)
4209{
4210 struct perf_event *event;
4211
4212 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4213 if (perf_event_task_match(event))
4214 perf_event_task_output(event, task_event);
4215 }
4216}
4217
4218static void perf_event_task_event(struct perf_task_event *task_event)
4219{
4220 struct perf_cpu_context *cpuctx;
4221 struct perf_event_context *ctx;
4222 struct pmu *pmu;
4223 int ctxn;
4224
4225 rcu_read_lock();
4226 list_for_each_entry_rcu(pmu, &pmus, entry) {
4227 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4228 if (cpuctx->active_pmu != pmu)
4229 goto next;
4230 perf_event_task_ctx(&cpuctx->ctx, task_event);
4231
4232 ctx = task_event->task_ctx;
4233 if (!ctx) {
4234 ctxn = pmu->task_ctx_nr;
4235 if (ctxn < 0)
4236 goto next;
4237 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4238 }
4239 if (ctx)
4240 perf_event_task_ctx(ctx, task_event);
4241next:
4242 put_cpu_ptr(pmu->pmu_cpu_context);
4243 }
4244 rcu_read_unlock();
4245}
4246
4247static void perf_event_task(struct task_struct *task,
4248 struct perf_event_context *task_ctx,
4249 int new)
4250{
4251 struct perf_task_event task_event;
4252
4253 if (!atomic_read(&nr_comm_events) &&
4254 !atomic_read(&nr_mmap_events) &&
4255 !atomic_read(&nr_task_events))
4256 return;
4257
4258 task_event = (struct perf_task_event){
4259 .task = task,
4260 .task_ctx = task_ctx,
4261 .event_id = {
4262 .header = {
4263 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4264 .misc = 0,
4265 .size = sizeof(task_event.event_id),
4266 },
4267 /* .pid */
4268 /* .ppid */
4269 /* .tid */
4270 /* .ptid */
4271 .time = perf_clock(),
4272 },
4273 };
4274
4275 perf_event_task_event(&task_event);
4276}
4277
4278void perf_event_fork(struct task_struct *task)
4279{
4280 perf_event_task(task, NULL, 1);
4281}
4282
4283/*
4284 * comm tracking
4285 */
4286
4287struct perf_comm_event {
4288 struct task_struct *task;
4289 char *comm;
4290 int comm_size;
4291
4292 struct {
4293 struct perf_event_header header;
4294
4295 u32 pid;
4296 u32 tid;
4297 } event_id;
4298};
4299
4300static void perf_event_comm_output(struct perf_event *event,
4301 struct perf_comm_event *comm_event)
4302{
4303 struct perf_output_handle handle;
4304 struct perf_sample_data sample;
4305 int size = comm_event->event_id.header.size;
4306 int ret;
4307
4308 perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4309 ret = perf_output_begin(&handle, event,
4310 comm_event->event_id.header.size);
4311
4312 if (ret)
4313 goto out;
4314
4315 comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4316 comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4317
4318 perf_output_put(&handle, comm_event->event_id);
4319 __output_copy(&handle, comm_event->comm,
4320 comm_event->comm_size);
4321
4322 perf_event__output_id_sample(event, &handle, &sample);
4323
4324 perf_output_end(&handle);
4325out:
4326 comm_event->event_id.header.size = size;
4327}
4328
4329static int perf_event_comm_match(struct perf_event *event)
4330{
4331 if (event->state < PERF_EVENT_STATE_INACTIVE)
4332 return 0;
4333
4334 if (!event_filter_match(event))
4335 return 0;
4336
4337 if (event->attr.comm)
4338 return 1;
4339
4340 return 0;
4341}
4342
4343static void perf_event_comm_ctx(struct perf_event_context *ctx,
4344 struct perf_comm_event *comm_event)
4345{
4346 struct perf_event *event;
4347
4348 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4349 if (perf_event_comm_match(event))
4350 perf_event_comm_output(event, comm_event);
4351 }
4352}
4353
4354static void perf_event_comm_event(struct perf_comm_event *comm_event)
4355{
4356 struct perf_cpu_context *cpuctx;
4357 struct perf_event_context *ctx;
4358 char comm[TASK_COMM_LEN];
4359 unsigned int size;
4360 struct pmu *pmu;
4361 int ctxn;
4362
4363 memset(comm, 0, sizeof(comm));
4364 strlcpy(comm, comm_event->task->comm, sizeof(comm));
4365 size = ALIGN(strlen(comm)+1, sizeof(u64));
4366
4367 comm_event->comm = comm;
4368 comm_event->comm_size = size;
4369
4370 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4371 rcu_read_lock();
4372 list_for_each_entry_rcu(pmu, &pmus, entry) {
4373 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4374 if (cpuctx->active_pmu != pmu)
4375 goto next;
4376 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
4377
4378 ctxn = pmu->task_ctx_nr;
4379 if (ctxn < 0)
4380 goto next;
4381
4382 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4383 if (ctx)
4384 perf_event_comm_ctx(ctx, comm_event);
4385next:
4386 put_cpu_ptr(pmu->pmu_cpu_context);
4387 }
4388 rcu_read_unlock();
4389}
4390
4391void perf_event_comm(struct task_struct *task)
4392{
4393 struct perf_comm_event comm_event;
4394 struct perf_event_context *ctx;
4395 int ctxn;
4396
4397 for_each_task_context_nr(ctxn) {
4398 ctx = task->perf_event_ctxp[ctxn];
4399 if (!ctx)
4400 continue;
4401
4402 perf_event_enable_on_exec(ctx);
4403 }
4404
4405 if (!atomic_read(&nr_comm_events))
4406 return;
4407
4408 comm_event = (struct perf_comm_event){
4409 .task = task,
4410 /* .comm */
4411 /* .comm_size */
4412 .event_id = {
4413 .header = {
4414 .type = PERF_RECORD_COMM,
4415 .misc = 0,
4416 /* .size */
4417 },
4418 /* .pid */
4419 /* .tid */
4420 },
4421 };
4422
4423 perf_event_comm_event(&comm_event);
4424}
4425
4426/*
4427 * mmap tracking
4428 */
4429
4430struct perf_mmap_event {
4431 struct vm_area_struct *vma;
4432
4433 const char *file_name;
4434 int file_size;
4435
4436 struct {
4437 struct perf_event_header header;
4438
4439 u32 pid;
4440 u32 tid;
4441 u64 start;
4442 u64 len;
4443 u64 pgoff;
4444 } event_id;
4445};
4446
4447static void perf_event_mmap_output(struct perf_event *event,
4448 struct perf_mmap_event *mmap_event)
4449{
4450 struct perf_output_handle handle;
4451 struct perf_sample_data sample;
4452 int size = mmap_event->event_id.header.size;
4453 int ret;
4454
4455 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4456 ret = perf_output_begin(&handle, event,
4457 mmap_event->event_id.header.size);
4458 if (ret)
4459 goto out;
4460
4461 mmap_event->event_id.pid = perf_event_pid(event, current);
4462 mmap_event->event_id.tid = perf_event_tid(event, current);
4463
4464 perf_output_put(&handle, mmap_event->event_id);
4465 __output_copy(&handle, mmap_event->file_name,
4466 mmap_event->file_size);
4467
4468 perf_event__output_id_sample(event, &handle, &sample);
4469
4470 perf_output_end(&handle);
4471out:
4472 mmap_event->event_id.header.size = size;
4473}
4474
4475static int perf_event_mmap_match(struct perf_event *event,
4476 struct perf_mmap_event *mmap_event,
4477 int executable)
4478{
4479 if (event->state < PERF_EVENT_STATE_INACTIVE)
4480 return 0;
4481
4482 if (!event_filter_match(event))
4483 return 0;
4484
4485 if ((!executable && event->attr.mmap_data) ||
4486 (executable && event->attr.mmap))
4487 return 1;
4488
4489 return 0;
4490}
4491
4492static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4493 struct perf_mmap_event *mmap_event,
4494 int executable)
4495{
4496 struct perf_event *event;
4497
4498 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4499 if (perf_event_mmap_match(event, mmap_event, executable))
4500 perf_event_mmap_output(event, mmap_event);
4501 }
4502}
4503
4504static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4505{
4506 struct perf_cpu_context *cpuctx;
4507 struct perf_event_context *ctx;
4508 struct vm_area_struct *vma = mmap_event->vma;
4509 struct file *file = vma->vm_file;
4510 unsigned int size;
4511 char tmp[16];
4512 char *buf = NULL;
4513 const char *name;
4514 struct pmu *pmu;
4515 int ctxn;
4516
4517 memset(tmp, 0, sizeof(tmp));
4518
4519 if (file) {
4520 /*
4521 * d_path works from the end of the rb backwards, so we
4522 * need to add enough zero bytes after the string to handle
4523 * the 64bit alignment we do later.
4524 */
4525 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4526 if (!buf) {
4527 name = strncpy(tmp, "//enomem", sizeof(tmp));
4528 goto got_name;
4529 }
4530 name = d_path(&file->f_path, buf, PATH_MAX);
4531 if (IS_ERR(name)) {
4532 name = strncpy(tmp, "//toolong", sizeof(tmp));
4533 goto got_name;
4534 }
4535 } else {
4536 if (arch_vma_name(mmap_event->vma)) {
4537 name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4538 sizeof(tmp));
4539 goto got_name;
4540 }
4541
4542 if (!vma->vm_mm) {
4543 name = strncpy(tmp, "[vdso]", sizeof(tmp));
4544 goto got_name;
4545 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4546 vma->vm_end >= vma->vm_mm->brk) {
4547 name = strncpy(tmp, "[heap]", sizeof(tmp));
4548 goto got_name;
4549 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4550 vma->vm_end >= vma->vm_mm->start_stack) {
4551 name = strncpy(tmp, "[stack]", sizeof(tmp));
4552 goto got_name;
4553 }
4554
4555 name = strncpy(tmp, "//anon", sizeof(tmp));
4556 goto got_name;
4557 }
4558
4559got_name:
4560 size = ALIGN(strlen(name)+1, sizeof(u64));
4561
4562 mmap_event->file_name = name;
4563 mmap_event->file_size = size;
4564
4565 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4566
4567 rcu_read_lock();
4568 list_for_each_entry_rcu(pmu, &pmus, entry) {
4569 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4570 if (cpuctx->active_pmu != pmu)
4571 goto next;
4572 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4573 vma->vm_flags & VM_EXEC);
4574
4575 ctxn = pmu->task_ctx_nr;
4576 if (ctxn < 0)
4577 goto next;
4578
4579 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4580 if (ctx) {
4581 perf_event_mmap_ctx(ctx, mmap_event,
4582 vma->vm_flags & VM_EXEC);
4583 }
4584next:
4585 put_cpu_ptr(pmu->pmu_cpu_context);
4586 }
4587 rcu_read_unlock();
4588
4589 kfree(buf);
4590}
4591
4592void perf_event_mmap(struct vm_area_struct *vma)
4593{
4594 struct perf_mmap_event mmap_event;
4595
4596 if (!atomic_read(&nr_mmap_events))
4597 return;
4598
4599 mmap_event = (struct perf_mmap_event){
4600 .vma = vma,
4601 /* .file_name */
4602 /* .file_size */
4603 .event_id = {
4604 .header = {
4605 .type = PERF_RECORD_MMAP,
4606 .misc = PERF_RECORD_MISC_USER,
4607 /* .size */
4608 },
4609 /* .pid */
4610 /* .tid */
4611 .start = vma->vm_start,
4612 .len = vma->vm_end - vma->vm_start,
4613 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
4614 },
4615 };
4616
4617 perf_event_mmap_event(&mmap_event);
4618}
4619
4620/*
4621 * IRQ throttle logging
4622 */
4623
4624static void perf_log_throttle(struct perf_event *event, int enable)
4625{
4626 struct perf_output_handle handle;
4627 struct perf_sample_data sample;
4628 int ret;
4629
4630 struct {
4631 struct perf_event_header header;
4632 u64 time;
4633 u64 id;
4634 u64 stream_id;
4635 } throttle_event = {
4636 .header = {
4637 .type = PERF_RECORD_THROTTLE,
4638 .misc = 0,
4639 .size = sizeof(throttle_event),
4640 },
4641 .time = perf_clock(),
4642 .id = primary_event_id(event),
4643 .stream_id = event->id,
4644 };
4645
4646 if (enable)
4647 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4648
4649 perf_event_header__init_id(&throttle_event.header, &sample, event);
4650
4651 ret = perf_output_begin(&handle, event,
4652 throttle_event.header.size);
4653 if (ret)
4654 return;
4655
4656 perf_output_put(&handle, throttle_event);
4657 perf_event__output_id_sample(event, &handle, &sample);
4658 perf_output_end(&handle);
4659}
4660
4661/*
4662 * Generic event overflow handling, sampling.
4663 */
4664
4665static int __perf_event_overflow(struct perf_event *event,
4666 int throttle, struct perf_sample_data *data,
4667 struct pt_regs *regs)
4668{
4669 int events = atomic_read(&event->event_limit);
4670 struct hw_perf_event *hwc = &event->hw;
4671 u64 seq;
4672 int ret = 0;
4673
4674 /*
4675 * Non-sampling counters might still use the PMI to fold short
4676 * hardware counters, ignore those.
4677 */
4678 if (unlikely(!is_sampling_event(event)))
4679 return 0;
4680
4681 seq = __this_cpu_read(perf_throttled_seq);
4682 if (seq != hwc->interrupts_seq) {
4683 hwc->interrupts_seq = seq;
4684 hwc->interrupts = 1;
4685 } else {
4686 hwc->interrupts++;
4687 if (unlikely(throttle
4688 && hwc->interrupts >= max_samples_per_tick)) {
4689 __this_cpu_inc(perf_throttled_count);
4690 hwc->interrupts = MAX_INTERRUPTS;
4691 perf_log_throttle(event, 0);
4692 ret = 1;
4693 }
4694 }
4695
4696 if (event->attr.freq) {
4697 u64 now = perf_clock();
4698 s64 delta = now - hwc->freq_time_stamp;
4699
4700 hwc->freq_time_stamp = now;
4701
4702 if (delta > 0 && delta < 2*TICK_NSEC)
4703 perf_adjust_period(event, delta, hwc->last_period, true);
4704 }
4705
4706 /*
4707 * XXX event_limit might not quite work as expected on inherited
4708 * events
4709 */
4710
4711 event->pending_kill = POLL_IN;
4712 if (events && atomic_dec_and_test(&event->event_limit)) {
4713 ret = 1;
4714 event->pending_kill = POLL_HUP;
4715 event->pending_disable = 1;
4716 irq_work_queue(&event->pending);
4717 }
4718
4719 if (event->overflow_handler)
4720 event->overflow_handler(event, data, regs);
4721 else
4722 perf_event_output(event, data, regs);
4723
4724 if (event->fasync && event->pending_kill) {
4725 event->pending_wakeup = 1;
4726 irq_work_queue(&event->pending);
4727 }
4728
4729 return ret;
4730}
4731
4732int perf_event_overflow(struct perf_event *event,
4733 struct perf_sample_data *data,
4734 struct pt_regs *regs)
4735{
4736 return __perf_event_overflow(event, 1, data, regs);
4737}
4738
4739/*
4740 * Generic software event infrastructure
4741 */
4742
4743struct swevent_htable {
4744 struct swevent_hlist *swevent_hlist;
4745 struct mutex hlist_mutex;
4746 int hlist_refcount;
4747
4748 /* Recursion avoidance in each contexts */
4749 int recursion[PERF_NR_CONTEXTS];
4750};
4751
4752static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
4753
4754/*
4755 * We directly increment event->count and keep a second value in
4756 * event->hw.period_left to count intervals. This period event
4757 * is kept in the range [-sample_period, 0] so that we can use the
4758 * sign as trigger.
4759 */
4760
4761static u64 perf_swevent_set_period(struct perf_event *event)
4762{
4763 struct hw_perf_event *hwc = &event->hw;
4764 u64 period = hwc->last_period;
4765 u64 nr, offset;
4766 s64 old, val;
4767
4768 hwc->last_period = hwc->sample_period;
4769
4770again:
4771 old = val = local64_read(&hwc->period_left);
4772 if (val < 0)
4773 return 0;
4774
4775 nr = div64_u64(period + val, period);
4776 offset = nr * period;
4777 val -= offset;
4778 if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4779 goto again;
4780
4781 return nr;
4782}
4783
4784static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4785 struct perf_sample_data *data,
4786 struct pt_regs *regs)
4787{
4788 struct hw_perf_event *hwc = &event->hw;
4789 int throttle = 0;
4790
4791 if (!overflow)
4792 overflow = perf_swevent_set_period(event);
4793
4794 if (hwc->interrupts == MAX_INTERRUPTS)
4795 return;
4796
4797 for (; overflow; overflow--) {
4798 if (__perf_event_overflow(event, throttle,
4799 data, regs)) {
4800 /*
4801 * We inhibit the overflow from happening when
4802 * hwc->interrupts == MAX_INTERRUPTS.
4803 */
4804 break;
4805 }
4806 throttle = 1;
4807 }
4808}
4809
4810static void perf_swevent_event(struct perf_event *event, u64 nr,
4811 struct perf_sample_data *data,
4812 struct pt_regs *regs)
4813{
4814 struct hw_perf_event *hwc = &event->hw;
4815
4816 local64_add(nr, &event->count);
4817
4818 if (!regs)
4819 return;
4820
4821 if (!is_sampling_event(event))
4822 return;
4823
4824 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
4825 data->period = nr;
4826 return perf_swevent_overflow(event, 1, data, regs);
4827 } else
4828 data->period = event->hw.last_period;
4829
4830 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4831 return perf_swevent_overflow(event, 1, data, regs);
4832
4833 if (local64_add_negative(nr, &hwc->period_left))
4834 return;
4835
4836 perf_swevent_overflow(event, 0, data, regs);
4837}
4838
4839static int perf_exclude_event(struct perf_event *event,
4840 struct pt_regs *regs)
4841{
4842 if (event->hw.state & PERF_HES_STOPPED)
4843 return 1;
4844
4845 if (regs) {
4846 if (event->attr.exclude_user && user_mode(regs))
4847 return 1;
4848
4849 if (event->attr.exclude_kernel && !user_mode(regs))
4850 return 1;
4851 }
4852
4853 return 0;
4854}
4855
4856static int perf_swevent_match(struct perf_event *event,
4857 enum perf_type_id type,
4858 u32 event_id,
4859 struct perf_sample_data *data,
4860 struct pt_regs *regs)
4861{
4862 if (event->attr.type != type)
4863 return 0;
4864
4865 if (event->attr.config != event_id)
4866 return 0;
4867
4868 if (perf_exclude_event(event, regs))
4869 return 0;
4870
4871 return 1;
4872}
4873
4874static inline u64 swevent_hash(u64 type, u32 event_id)
4875{
4876 u64 val = event_id | (type << 32);
4877
4878 return hash_64(val, SWEVENT_HLIST_BITS);
4879}
4880
4881static inline struct hlist_head *
4882__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4883{
4884 u64 hash = swevent_hash(type, event_id);
4885
4886 return &hlist->heads[hash];
4887}
4888
4889/* For the read side: events when they trigger */
4890static inline struct hlist_head *
4891find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4892{
4893 struct swevent_hlist *hlist;
4894
4895 hlist = rcu_dereference(swhash->swevent_hlist);
4896 if (!hlist)
4897 return NULL;
4898
4899 return __find_swevent_head(hlist, type, event_id);
4900}
4901
4902/* For the event head insertion and removal in the hlist */
4903static inline struct hlist_head *
4904find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4905{
4906 struct swevent_hlist *hlist;
4907 u32 event_id = event->attr.config;
4908 u64 type = event->attr.type;
4909
4910 /*
4911 * Event scheduling is always serialized against hlist allocation
4912 * and release. Which makes the protected version suitable here.
4913 * The context lock guarantees that.
4914 */
4915 hlist = rcu_dereference_protected(swhash->swevent_hlist,
4916 lockdep_is_held(&event->ctx->lock));
4917 if (!hlist)
4918 return NULL;
4919
4920 return __find_swevent_head(hlist, type, event_id);
4921}
4922
4923static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4924 u64 nr,
4925 struct perf_sample_data *data,
4926 struct pt_regs *regs)
4927{
4928 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4929 struct perf_event *event;
4930 struct hlist_node *node;
4931 struct hlist_head *head;
4932
4933 rcu_read_lock();
4934 head = find_swevent_head_rcu(swhash, type, event_id);
4935 if (!head)
4936 goto end;
4937
4938 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4939 if (perf_swevent_match(event, type, event_id, data, regs))
4940 perf_swevent_event(event, nr, data, regs);
4941 }
4942end:
4943 rcu_read_unlock();
4944}
4945
4946int perf_swevent_get_recursion_context(void)
4947{
4948 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4949
4950 return get_recursion_context(swhash->recursion);
4951}
4952EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
4953
4954inline void perf_swevent_put_recursion_context(int rctx)
4955{
4956 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4957
4958 put_recursion_context(swhash->recursion, rctx);
4959}
4960
4961void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
4962{
4963 struct perf_sample_data data;
4964 int rctx;
4965
4966 preempt_disable_notrace();
4967 rctx = perf_swevent_get_recursion_context();
4968 if (rctx < 0)
4969 return;
4970
4971 perf_sample_data_init(&data, addr, 0);
4972
4973 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
4974
4975 perf_swevent_put_recursion_context(rctx);
4976 preempt_enable_notrace();
4977}
4978
4979static void perf_swevent_read(struct perf_event *event)
4980{
4981}
4982
4983static int perf_swevent_add(struct perf_event *event, int flags)
4984{
4985 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4986 struct hw_perf_event *hwc = &event->hw;
4987 struct hlist_head *head;
4988
4989 if (is_sampling_event(event)) {
4990 hwc->last_period = hwc->sample_period;
4991 perf_swevent_set_period(event);
4992 }
4993
4994 hwc->state = !(flags & PERF_EF_START);
4995
4996 head = find_swevent_head(swhash, event);
4997 if (WARN_ON_ONCE(!head))
4998 return -EINVAL;
4999
5000 hlist_add_head_rcu(&event->hlist_entry, head);
5001
5002 return 0;
5003}
5004
5005static void perf_swevent_del(struct perf_event *event, int flags)
5006{
5007 hlist_del_rcu(&event->hlist_entry);
5008}
5009
5010static void perf_swevent_start(struct perf_event *event, int flags)
5011{
5012 event->hw.state = 0;
5013}
5014
5015static void perf_swevent_stop(struct perf_event *event, int flags)
5016{
5017 event->hw.state = PERF_HES_STOPPED;
5018}
5019
5020/* Deref the hlist from the update side */
5021static inline struct swevent_hlist *
5022swevent_hlist_deref(struct swevent_htable *swhash)
5023{
5024 return rcu_dereference_protected(swhash->swevent_hlist,
5025 lockdep_is_held(&swhash->hlist_mutex));
5026}
5027
5028static void swevent_hlist_release(struct swevent_htable *swhash)
5029{
5030 struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5031
5032 if (!hlist)
5033 return;
5034
5035 rcu_assign_pointer(swhash->swevent_hlist, NULL);
5036 kfree_rcu(hlist, rcu_head);
5037}
5038
5039static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5040{
5041 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5042
5043 mutex_lock(&swhash->hlist_mutex);
5044
5045 if (!--swhash->hlist_refcount)
5046 swevent_hlist_release(swhash);
5047
5048 mutex_unlock(&swhash->hlist_mutex);
5049}
5050
5051static void swevent_hlist_put(struct perf_event *event)
5052{
5053 int cpu;
5054
5055 if (event->cpu != -1) {
5056 swevent_hlist_put_cpu(event, event->cpu);
5057 return;
5058 }
5059
5060 for_each_possible_cpu(cpu)
5061 swevent_hlist_put_cpu(event, cpu);
5062}
5063
5064static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
5065{
5066 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5067 int err = 0;
5068
5069 mutex_lock(&swhash->hlist_mutex);
5070
5071 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5072 struct swevent_hlist *hlist;
5073
5074 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5075 if (!hlist) {
5076 err = -ENOMEM;
5077 goto exit;
5078 }
5079 rcu_assign_pointer(swhash->swevent_hlist, hlist);
5080 }
5081 swhash->hlist_refcount++;
5082exit:
5083 mutex_unlock(&swhash->hlist_mutex);
5084
5085 return err;
5086}
5087
5088static int swevent_hlist_get(struct perf_event *event)
5089{
5090 int err;
5091 int cpu, failed_cpu;
5092
5093 if (event->cpu != -1)
5094 return swevent_hlist_get_cpu(event, event->cpu);
5095
5096 get_online_cpus();
5097 for_each_possible_cpu(cpu) {
5098 err = swevent_hlist_get_cpu(event, cpu);
5099 if (err) {
5100 failed_cpu = cpu;
5101 goto fail;
5102 }
5103 }
5104 put_online_cpus();
5105
5106 return 0;
5107fail:
5108 for_each_possible_cpu(cpu) {
5109 if (cpu == failed_cpu)
5110 break;
5111 swevent_hlist_put_cpu(event, cpu);
5112 }
5113
5114 put_online_cpus();
5115 return err;
5116}
5117
5118struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
5119
5120static void sw_perf_event_destroy(struct perf_event *event)
5121{
5122 u64 event_id = event->attr.config;
5123
5124 WARN_ON(event->parent);
5125
5126 static_key_slow_dec(&perf_swevent_enabled[event_id]);
5127 swevent_hlist_put(event);
5128}
5129
5130static int perf_swevent_init(struct perf_event *event)
5131{
5132 int event_id = event->attr.config;
5133
5134 if (event->attr.type != PERF_TYPE_SOFTWARE)
5135 return -ENOENT;
5136
5137 /*
5138 * no branch sampling for software events
5139 */
5140 if (has_branch_stack(event))
5141 return -EOPNOTSUPP;
5142
5143 switch (event_id) {
5144 case PERF_COUNT_SW_CPU_CLOCK:
5145 case PERF_COUNT_SW_TASK_CLOCK:
5146 return -ENOENT;
5147
5148 default:
5149 break;
5150 }
5151
5152 if (event_id >= PERF_COUNT_SW_MAX)
5153 return -ENOENT;
5154
5155 if (!event->parent) {
5156 int err;
5157
5158 err = swevent_hlist_get(event);
5159 if (err)
5160 return err;
5161
5162 static_key_slow_inc(&perf_swevent_enabled[event_id]);
5163 event->destroy = sw_perf_event_destroy;
5164 }
5165
5166 return 0;
5167}
5168
5169static int perf_swevent_event_idx(struct perf_event *event)
5170{
5171 return 0;
5172}
5173
5174static struct pmu perf_swevent = {
5175 .task_ctx_nr = perf_sw_context,
5176
5177 .event_init = perf_swevent_init,
5178 .add = perf_swevent_add,
5179 .del = perf_swevent_del,
5180 .start = perf_swevent_start,
5181 .stop = perf_swevent_stop,
5182 .read = perf_swevent_read,
5183
5184 .event_idx = perf_swevent_event_idx,
5185};
5186
5187#ifdef CONFIG_EVENT_TRACING
5188
5189static int perf_tp_filter_match(struct perf_event *event,
5190 struct perf_sample_data *data)
5191{
5192 void *record = data->raw->data;
5193
5194 if (likely(!event->filter) || filter_match_preds(event->filter, record))
5195 return 1;
5196 return 0;
5197}
5198
5199static int perf_tp_event_match(struct perf_event *event,
5200 struct perf_sample_data *data,
5201 struct pt_regs *regs)
5202{
5203 if (event->hw.state & PERF_HES_STOPPED)
5204 return 0;
5205 /*
5206 * All tracepoints are from kernel-space.
5207 */
5208 if (event->attr.exclude_kernel)
5209 return 0;
5210
5211 if (!perf_tp_filter_match(event, data))
5212 return 0;
5213
5214 return 1;
5215}
5216
5217void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5218 struct pt_regs *regs, struct hlist_head *head, int rctx)
5219{
5220 struct perf_sample_data data;
5221 struct perf_event *event;
5222 struct hlist_node *node;
5223
5224 struct perf_raw_record raw = {
5225 .size = entry_size,
5226 .data = record,
5227 };
5228
5229 perf_sample_data_init(&data, addr, 0);
5230 data.raw = &raw;
5231
5232 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5233 if (perf_tp_event_match(event, &data, regs))
5234 perf_swevent_event(event, count, &data, regs);
5235 }
5236
5237 perf_swevent_put_recursion_context(rctx);
5238}
5239EXPORT_SYMBOL_GPL(perf_tp_event);
5240
5241static void tp_perf_event_destroy(struct perf_event *event)
5242{
5243 perf_trace_destroy(event);
5244}
5245
5246static int perf_tp_event_init(struct perf_event *event)
5247{
5248 int err;
5249
5250 if (event->attr.type != PERF_TYPE_TRACEPOINT)
5251 return -ENOENT;
5252
5253 /*
5254 * no branch sampling for tracepoint events
5255 */
5256 if (has_branch_stack(event))
5257 return -EOPNOTSUPP;
5258
5259 err = perf_trace_init(event);
5260 if (err)
5261 return err;
5262
5263 event->destroy = tp_perf_event_destroy;
5264
5265 return 0;
5266}
5267
5268static struct pmu perf_tracepoint = {
5269 .task_ctx_nr = perf_sw_context,
5270
5271 .event_init = perf_tp_event_init,
5272 .add = perf_trace_add,
5273 .del = perf_trace_del,
5274 .start = perf_swevent_start,
5275 .stop = perf_swevent_stop,
5276 .read = perf_swevent_read,
5277
5278 .event_idx = perf_swevent_event_idx,
5279};
5280
5281static inline void perf_tp_register(void)
5282{
5283 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
5284}
5285
5286static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5287{
5288 char *filter_str;
5289 int ret;
5290
5291 if (event->attr.type != PERF_TYPE_TRACEPOINT)
5292 return -EINVAL;
5293
5294 filter_str = strndup_user(arg, PAGE_SIZE);
5295 if (IS_ERR(filter_str))
5296 return PTR_ERR(filter_str);
5297
5298 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
5299
5300 kfree(filter_str);
5301 return ret;
5302}
5303
5304static void perf_event_free_filter(struct perf_event *event)
5305{
5306 ftrace_profile_free_filter(event);
5307}
5308
5309#else
5310
5311static inline void perf_tp_register(void)
5312{
5313}
5314
5315static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5316{
5317 return -ENOENT;
5318}
5319
5320static void perf_event_free_filter(struct perf_event *event)
5321{
5322}
5323
5324#endif /* CONFIG_EVENT_TRACING */
5325
5326#ifdef CONFIG_HAVE_HW_BREAKPOINT
5327void perf_bp_event(struct perf_event *bp, void *data)
5328{
5329 struct perf_sample_data sample;
5330 struct pt_regs *regs = data;
5331
5332 perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
5333
5334 if (!bp->hw.state && !perf_exclude_event(bp, regs))
5335 perf_swevent_event(bp, 1, &sample, regs);
5336}
5337#endif
5338
5339/*
5340 * hrtimer based swevent callback
5341 */
5342
5343static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5344{
5345 enum hrtimer_restart ret = HRTIMER_RESTART;
5346 struct perf_sample_data data;
5347 struct pt_regs *regs;
5348 struct perf_event *event;
5349 u64 period;
5350
5351 event = container_of(hrtimer, struct perf_event, hw.hrtimer);
5352
5353 if (event->state != PERF_EVENT_STATE_ACTIVE)
5354 return HRTIMER_NORESTART;
5355
5356 event->pmu->read(event);
5357
5358 perf_sample_data_init(&data, 0, event->hw.last_period);
5359 regs = get_irq_regs();
5360
5361 if (regs && !perf_exclude_event(event, regs)) {
5362 if (!(event->attr.exclude_idle && is_idle_task(current)))
5363 if (__perf_event_overflow(event, 1, &data, regs))
5364 ret = HRTIMER_NORESTART;
5365 }
5366
5367 period = max_t(u64, 10000, event->hw.sample_period);
5368 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5369
5370 return ret;
5371}
5372
5373static void perf_swevent_start_hrtimer(struct perf_event *event)
5374{
5375 struct hw_perf_event *hwc = &event->hw;
5376 s64 period;
5377
5378 if (!is_sampling_event(event))
5379 return;
5380
5381 period = local64_read(&hwc->period_left);
5382 if (period) {
5383 if (period < 0)
5384 period = 10000;
5385
5386 local64_set(&hwc->period_left, 0);
5387 } else {
5388 period = max_t(u64, 10000, hwc->sample_period);
5389 }
5390 __hrtimer_start_range_ns(&hwc->hrtimer,
5391 ns_to_ktime(period), 0,
5392 HRTIMER_MODE_REL_PINNED, 0);
5393}
5394
5395static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5396{
5397 struct hw_perf_event *hwc = &event->hw;
5398
5399 if (is_sampling_event(event)) {
5400 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
5401 local64_set(&hwc->period_left, ktime_to_ns(remaining));
5402
5403 hrtimer_cancel(&hwc->hrtimer);
5404 }
5405}
5406
5407static void perf_swevent_init_hrtimer(struct perf_event *event)
5408{
5409 struct hw_perf_event *hwc = &event->hw;
5410
5411 if (!is_sampling_event(event))
5412 return;
5413
5414 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5415 hwc->hrtimer.function = perf_swevent_hrtimer;
5416
5417 /*
5418 * Since hrtimers have a fixed rate, we can do a static freq->period
5419 * mapping and avoid the whole period adjust feedback stuff.
5420 */
5421 if (event->attr.freq) {
5422 long freq = event->attr.sample_freq;
5423
5424 event->attr.sample_period = NSEC_PER_SEC / freq;
5425 hwc->sample_period = event->attr.sample_period;
5426 local64_set(&hwc->period_left, hwc->sample_period);
5427 event->attr.freq = 0;
5428 }
5429}
5430
5431/*
5432 * Software event: cpu wall time clock
5433 */
5434
5435static void cpu_clock_event_update(struct perf_event *event)
5436{
5437 s64 prev;
5438 u64 now;
5439
5440 now = local_clock();
5441 prev = local64_xchg(&event->hw.prev_count, now);
5442 local64_add(now - prev, &event->count);
5443}
5444
5445static void cpu_clock_event_start(struct perf_event *event, int flags)
5446{
5447 local64_set(&event->hw.prev_count, local_clock());
5448 perf_swevent_start_hrtimer(event);
5449}
5450
5451static void cpu_clock_event_stop(struct perf_event *event, int flags)
5452{
5453 perf_swevent_cancel_hrtimer(event);
5454 cpu_clock_event_update(event);
5455}
5456
5457static int cpu_clock_event_add(struct perf_event *event, int flags)
5458{
5459 if (flags & PERF_EF_START)
5460 cpu_clock_event_start(event, flags);
5461
5462 return 0;
5463}
5464
5465static void cpu_clock_event_del(struct perf_event *event, int flags)
5466{
5467 cpu_clock_event_stop(event, flags);
5468}
5469
5470static void cpu_clock_event_read(struct perf_event *event)
5471{
5472 cpu_clock_event_update(event);
5473}
5474
5475static int cpu_clock_event_init(struct perf_event *event)
5476{
5477 if (event->attr.type != PERF_TYPE_SOFTWARE)
5478 return -ENOENT;
5479
5480 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5481 return -ENOENT;
5482
5483 /*
5484 * no branch sampling for software events
5485 */
5486 if (has_branch_stack(event))
5487 return -EOPNOTSUPP;
5488
5489 perf_swevent_init_hrtimer(event);
5490
5491 return 0;
5492}
5493
5494static struct pmu perf_cpu_clock = {
5495 .task_ctx_nr = perf_sw_context,
5496
5497 .event_init = cpu_clock_event_init,
5498 .add = cpu_clock_event_add,
5499 .del = cpu_clock_event_del,
5500 .start = cpu_clock_event_start,
5501 .stop = cpu_clock_event_stop,
5502 .read = cpu_clock_event_read,
5503
5504 .event_idx = perf_swevent_event_idx,
5505};
5506
5507/*
5508 * Software event: task time clock
5509 */
5510
5511static void task_clock_event_update(struct perf_event *event, u64 now)
5512{
5513 u64 prev;
5514 s64 delta;
5515
5516 prev = local64_xchg(&event->hw.prev_count, now);
5517 delta = now - prev;
5518 local64_add(delta, &event->count);
5519}
5520
5521static void task_clock_event_start(struct perf_event *event, int flags)
5522{
5523 local64_set(&event->hw.prev_count, event->ctx->time);
5524 perf_swevent_start_hrtimer(event);
5525}
5526
5527static void task_clock_event_stop(struct perf_event *event, int flags)
5528{
5529 perf_swevent_cancel_hrtimer(event);
5530 task_clock_event_update(event, event->ctx->time);
5531}
5532
5533static int task_clock_event_add(struct perf_event *event, int flags)
5534{
5535 if (flags & PERF_EF_START)
5536 task_clock_event_start(event, flags);
5537
5538 return 0;
5539}
5540
5541static void task_clock_event_del(struct perf_event *event, int flags)
5542{
5543 task_clock_event_stop(event, PERF_EF_UPDATE);
5544}
5545
5546static void task_clock_event_read(struct perf_event *event)
5547{
5548 u64 now = perf_clock();
5549 u64 delta = now - event->ctx->timestamp;
5550 u64 time = event->ctx->time + delta;
5551
5552 task_clock_event_update(event, time);
5553}
5554
5555static int task_clock_event_init(struct perf_event *event)
5556{
5557 if (event->attr.type != PERF_TYPE_SOFTWARE)
5558 return -ENOENT;
5559
5560 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5561 return -ENOENT;
5562
5563 /*
5564 * no branch sampling for software events
5565 */
5566 if (has_branch_stack(event))
5567 return -EOPNOTSUPP;
5568
5569 perf_swevent_init_hrtimer(event);
5570
5571 return 0;
5572}
5573
5574static struct pmu perf_task_clock = {
5575 .task_ctx_nr = perf_sw_context,
5576
5577 .event_init = task_clock_event_init,
5578 .add = task_clock_event_add,
5579 .del = task_clock_event_del,
5580 .start = task_clock_event_start,
5581 .stop = task_clock_event_stop,
5582 .read = task_clock_event_read,
5583
5584 .event_idx = perf_swevent_event_idx,
5585};
5586
5587static void perf_pmu_nop_void(struct pmu *pmu)
5588{
5589}
5590
5591static int perf_pmu_nop_int(struct pmu *pmu)
5592{
5593 return 0;
5594}
5595
5596static void perf_pmu_start_txn(struct pmu *pmu)
5597{
5598 perf_pmu_disable(pmu);
5599}
5600
5601static int perf_pmu_commit_txn(struct pmu *pmu)
5602{
5603 perf_pmu_enable(pmu);
5604 return 0;
5605}
5606
5607static void perf_pmu_cancel_txn(struct pmu *pmu)
5608{
5609 perf_pmu_enable(pmu);
5610}
5611
5612static int perf_event_idx_default(struct perf_event *event)
5613{
5614 return event->hw.idx + 1;
5615}
5616
5617/*
5618 * Ensures all contexts with the same task_ctx_nr have the same
5619 * pmu_cpu_context too.
5620 */
5621static void *find_pmu_context(int ctxn)
5622{
5623 struct pmu *pmu;
5624
5625 if (ctxn < 0)
5626 return NULL;
5627
5628 list_for_each_entry(pmu, &pmus, entry) {
5629 if (pmu->task_ctx_nr == ctxn)
5630 return pmu->pmu_cpu_context;
5631 }
5632
5633 return NULL;
5634}
5635
5636static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5637{
5638 int cpu;
5639
5640 for_each_possible_cpu(cpu) {
5641 struct perf_cpu_context *cpuctx;
5642
5643 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5644
5645 if (cpuctx->active_pmu == old_pmu)
5646 cpuctx->active_pmu = pmu;
5647 }
5648}
5649
5650static void free_pmu_context(struct pmu *pmu)
5651{
5652 struct pmu *i;
5653
5654 mutex_lock(&pmus_lock);
5655 /*
5656 * Like a real lame refcount.
5657 */
5658 list_for_each_entry(i, &pmus, entry) {
5659 if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5660 update_pmu_context(i, pmu);
5661 goto out;
5662 }
5663 }
5664
5665 free_percpu(pmu->pmu_cpu_context);
5666out:
5667 mutex_unlock(&pmus_lock);
5668}
5669static struct idr pmu_idr;
5670
5671static ssize_t
5672type_show(struct device *dev, struct device_attribute *attr, char *page)
5673{
5674 struct pmu *pmu = dev_get_drvdata(dev);
5675
5676 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
5677}
5678
5679static struct device_attribute pmu_dev_attrs[] = {
5680 __ATTR_RO(type),
5681 __ATTR_NULL,
5682};
5683
5684static int pmu_bus_running;
5685static struct bus_type pmu_bus = {
5686 .name = "event_source",
5687 .dev_attrs = pmu_dev_attrs,
5688};
5689
5690static void pmu_dev_release(struct device *dev)
5691{
5692 kfree(dev);
5693}
5694
5695static int pmu_dev_alloc(struct pmu *pmu)
5696{
5697 int ret = -ENOMEM;
5698
5699 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
5700 if (!pmu->dev)
5701 goto out;
5702
5703 pmu->dev->groups = pmu->attr_groups;
5704 device_initialize(pmu->dev);
5705 ret = dev_set_name(pmu->dev, "%s", pmu->name);
5706 if (ret)
5707 goto free_dev;
5708
5709 dev_set_drvdata(pmu->dev, pmu);
5710 pmu->dev->bus = &pmu_bus;
5711 pmu->dev->release = pmu_dev_release;
5712 ret = device_add(pmu->dev);
5713 if (ret)
5714 goto free_dev;
5715
5716out:
5717 return ret;
5718
5719free_dev:
5720 put_device(pmu->dev);
5721 goto out;
5722}
5723
5724static struct lock_class_key cpuctx_mutex;
5725static struct lock_class_key cpuctx_lock;
5726
5727int perf_pmu_register(struct pmu *pmu, char *name, int type)
5728{
5729 int cpu, ret;
5730
5731 mutex_lock(&pmus_lock);
5732 ret = -ENOMEM;
5733 pmu->pmu_disable_count = alloc_percpu(int);
5734 if (!pmu->pmu_disable_count)
5735 goto unlock;
5736
5737 pmu->type = -1;
5738 if (!name)
5739 goto skip_type;
5740 pmu->name = name;
5741
5742 if (type < 0) {
5743 int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
5744 if (!err)
5745 goto free_pdc;
5746
5747 err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
5748 if (err) {
5749 ret = err;
5750 goto free_pdc;
5751 }
5752 }
5753 pmu->type = type;
5754
5755 if (pmu_bus_running) {
5756 ret = pmu_dev_alloc(pmu);
5757 if (ret)
5758 goto free_idr;
5759 }
5760
5761skip_type:
5762 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
5763 if (pmu->pmu_cpu_context)
5764 goto got_cpu_context;
5765
5766 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
5767 if (!pmu->pmu_cpu_context)
5768 goto free_dev;
5769
5770 for_each_possible_cpu(cpu) {
5771 struct perf_cpu_context *cpuctx;
5772
5773 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5774 __perf_event_init_context(&cpuctx->ctx);
5775 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5776 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
5777 cpuctx->ctx.type = cpu_context;
5778 cpuctx->ctx.pmu = pmu;
5779 cpuctx->jiffies_interval = 1;
5780 INIT_LIST_HEAD(&cpuctx->rotation_list);
5781 cpuctx->active_pmu = pmu;
5782 }
5783
5784got_cpu_context:
5785 if (!pmu->start_txn) {
5786 if (pmu->pmu_enable) {
5787 /*
5788 * If we have pmu_enable/pmu_disable calls, install
5789 * transaction stubs that use that to try and batch
5790 * hardware accesses.
5791 */
5792 pmu->start_txn = perf_pmu_start_txn;
5793 pmu->commit_txn = perf_pmu_commit_txn;
5794 pmu->cancel_txn = perf_pmu_cancel_txn;
5795 } else {
5796 pmu->start_txn = perf_pmu_nop_void;
5797 pmu->commit_txn = perf_pmu_nop_int;
5798 pmu->cancel_txn = perf_pmu_nop_void;
5799 }
5800 }
5801
5802 if (!pmu->pmu_enable) {
5803 pmu->pmu_enable = perf_pmu_nop_void;
5804 pmu->pmu_disable = perf_pmu_nop_void;
5805 }
5806
5807 if (!pmu->event_idx)
5808 pmu->event_idx = perf_event_idx_default;
5809
5810 list_add_rcu(&pmu->entry, &pmus);
5811 ret = 0;
5812unlock:
5813 mutex_unlock(&pmus_lock);
5814
5815 return ret;
5816
5817free_dev:
5818 device_del(pmu->dev);
5819 put_device(pmu->dev);
5820
5821free_idr:
5822 if (pmu->type >= PERF_TYPE_MAX)
5823 idr_remove(&pmu_idr, pmu->type);
5824
5825free_pdc:
5826 free_percpu(pmu->pmu_disable_count);
5827 goto unlock;
5828}
5829
5830void perf_pmu_unregister(struct pmu *pmu)
5831{
5832 mutex_lock(&pmus_lock);
5833 list_del_rcu(&pmu->entry);
5834 mutex_unlock(&pmus_lock);
5835
5836 /*
5837 * We dereference the pmu list under both SRCU and regular RCU, so
5838 * synchronize against both of those.
5839 */
5840 synchronize_srcu(&pmus_srcu);
5841 synchronize_rcu();
5842
5843 free_percpu(pmu->pmu_disable_count);
5844 if (pmu->type >= PERF_TYPE_MAX)
5845 idr_remove(&pmu_idr, pmu->type);
5846 device_del(pmu->dev);
5847 put_device(pmu->dev);
5848 free_pmu_context(pmu);
5849}
5850
5851struct pmu *perf_init_event(struct perf_event *event)
5852{
5853 struct pmu *pmu = NULL;
5854 int idx;
5855 int ret;
5856
5857 idx = srcu_read_lock(&pmus_srcu);
5858
5859 rcu_read_lock();
5860 pmu = idr_find(&pmu_idr, event->attr.type);
5861 rcu_read_unlock();
5862 if (pmu) {
5863 event->pmu = pmu;
5864 ret = pmu->event_init(event);
5865 if (ret)
5866 pmu = ERR_PTR(ret);
5867 goto unlock;
5868 }
5869
5870 list_for_each_entry_rcu(pmu, &pmus, entry) {
5871 event->pmu = pmu;
5872 ret = pmu->event_init(event);
5873 if (!ret)
5874 goto unlock;
5875
5876 if (ret != -ENOENT) {
5877 pmu = ERR_PTR(ret);
5878 goto unlock;
5879 }
5880 }
5881 pmu = ERR_PTR(-ENOENT);
5882unlock:
5883 srcu_read_unlock(&pmus_srcu, idx);
5884
5885 return pmu;
5886}
5887
5888/*
5889 * Allocate and initialize a event structure
5890 */
5891static struct perf_event *
5892perf_event_alloc(struct perf_event_attr *attr, int cpu,
5893 struct task_struct *task,
5894 struct perf_event *group_leader,
5895 struct perf_event *parent_event,
5896 perf_overflow_handler_t overflow_handler,
5897 void *context)
5898{
5899 struct pmu *pmu;
5900 struct perf_event *event;
5901 struct hw_perf_event *hwc;
5902 long err;
5903
5904 if ((unsigned)cpu >= nr_cpu_ids) {
5905 if (!task || cpu != -1)
5906 return ERR_PTR(-EINVAL);
5907 }
5908
5909 event = kzalloc(sizeof(*event), GFP_KERNEL);
5910 if (!event)
5911 return ERR_PTR(-ENOMEM);
5912
5913 /*
5914 * Single events are their own group leaders, with an
5915 * empty sibling list:
5916 */
5917 if (!group_leader)
5918 group_leader = event;
5919
5920 mutex_init(&event->child_mutex);
5921 INIT_LIST_HEAD(&event->child_list);
5922
5923 INIT_LIST_HEAD(&event->group_entry);
5924 INIT_LIST_HEAD(&event->event_entry);
5925 INIT_LIST_HEAD(&event->sibling_list);
5926 INIT_LIST_HEAD(&event->rb_entry);
5927
5928 init_waitqueue_head(&event->waitq);
5929 init_irq_work(&event->pending, perf_pending_event);
5930
5931 mutex_init(&event->mmap_mutex);
5932
5933 atomic_long_set(&event->refcount, 1);
5934 event->cpu = cpu;
5935 event->attr = *attr;
5936 event->group_leader = group_leader;
5937 event->pmu = NULL;
5938 event->oncpu = -1;
5939
5940 event->parent = parent_event;
5941
5942 event->ns = get_pid_ns(current->nsproxy->pid_ns);
5943 event->id = atomic64_inc_return(&perf_event_id);
5944
5945 event->state = PERF_EVENT_STATE_INACTIVE;
5946
5947 if (task) {
5948 event->attach_state = PERF_ATTACH_TASK;
5949#ifdef CONFIG_HAVE_HW_BREAKPOINT
5950 /*
5951 * hw_breakpoint is a bit difficult here..
5952 */
5953 if (attr->type == PERF_TYPE_BREAKPOINT)
5954 event->hw.bp_target = task;
5955#endif
5956 }
5957
5958 if (!overflow_handler && parent_event) {
5959 overflow_handler = parent_event->overflow_handler;
5960 context = parent_event->overflow_handler_context;
5961 }
5962
5963 event->overflow_handler = overflow_handler;
5964 event->overflow_handler_context = context;
5965
5966 if (attr->disabled)
5967 event->state = PERF_EVENT_STATE_OFF;
5968
5969 pmu = NULL;
5970
5971 hwc = &event->hw;
5972 hwc->sample_period = attr->sample_period;
5973 if (attr->freq && attr->sample_freq)
5974 hwc->sample_period = 1;
5975 hwc->last_period = hwc->sample_period;
5976
5977 local64_set(&hwc->period_left, hwc->sample_period);
5978
5979 /*
5980 * we currently do not support PERF_FORMAT_GROUP on inherited events
5981 */
5982 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5983 goto done;
5984
5985 pmu = perf_init_event(event);
5986
5987done:
5988 err = 0;
5989 if (!pmu)
5990 err = -EINVAL;
5991 else if (IS_ERR(pmu))
5992 err = PTR_ERR(pmu);
5993
5994 if (err) {
5995 if (event->ns)
5996 put_pid_ns(event->ns);
5997 kfree(event);
5998 return ERR_PTR(err);
5999 }
6000
6001 if (!event->parent) {
6002 if (event->attach_state & PERF_ATTACH_TASK)
6003 static_key_slow_inc(&perf_sched_events.key);
6004 if (event->attr.mmap || event->attr.mmap_data)
6005 atomic_inc(&nr_mmap_events);
6006 if (event->attr.comm)
6007 atomic_inc(&nr_comm_events);
6008 if (event->attr.task)
6009 atomic_inc(&nr_task_events);
6010 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
6011 err = get_callchain_buffers();
6012 if (err) {
6013 free_event(event);
6014 return ERR_PTR(err);
6015 }
6016 }
6017 if (has_branch_stack(event)) {
6018 static_key_slow_inc(&perf_sched_events.key);
6019 if (!(event->attach_state & PERF_ATTACH_TASK))
6020 atomic_inc(&per_cpu(perf_branch_stack_events,
6021 event->cpu));
6022 }
6023 }
6024
6025 return event;
6026}
6027
6028static int perf_copy_attr(struct perf_event_attr __user *uattr,
6029 struct perf_event_attr *attr)
6030{
6031 u32 size;
6032 int ret;
6033
6034 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
6035 return -EFAULT;
6036
6037 /*
6038 * zero the full structure, so that a short copy will be nice.
6039 */
6040 memset(attr, 0, sizeof(*attr));
6041
6042 ret = get_user(size, &uattr->size);
6043 if (ret)
6044 return ret;
6045
6046 if (size > PAGE_SIZE) /* silly large */
6047 goto err_size;
6048
6049 if (!size) /* abi compat */
6050 size = PERF_ATTR_SIZE_VER0;
6051
6052 if (size < PERF_ATTR_SIZE_VER0)
6053 goto err_size;
6054
6055 /*
6056 * If we're handed a bigger struct than we know of,
6057 * ensure all the unknown bits are 0 - i.e. new
6058 * user-space does not rely on any kernel feature
6059 * extensions we dont know about yet.
6060 */
6061 if (size > sizeof(*attr)) {
6062 unsigned char __user *addr;
6063 unsigned char __user *end;
6064 unsigned char val;
6065
6066 addr = (void __user *)uattr + sizeof(*attr);
6067 end = (void __user *)uattr + size;
6068
6069 for (; addr < end; addr++) {
6070 ret = get_user(val, addr);
6071 if (ret)
6072 return ret;
6073 if (val)
6074 goto err_size;
6075 }
6076 size = sizeof(*attr);
6077 }
6078
6079 ret = copy_from_user(attr, uattr, size);
6080 if (ret)
6081 return -EFAULT;
6082
6083 if (attr->__reserved_1)
6084 return -EINVAL;
6085
6086 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
6087 return -EINVAL;
6088
6089 if (attr->read_format & ~(PERF_FORMAT_MAX-1))
6090 return -EINVAL;
6091
6092 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
6093 u64 mask = attr->branch_sample_type;
6094
6095 /* only using defined bits */
6096 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
6097 return -EINVAL;
6098
6099 /* at least one branch bit must be set */
6100 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
6101 return -EINVAL;
6102
6103 /* kernel level capture: check permissions */
6104 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
6105 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6106 return -EACCES;
6107
6108 /* propagate priv level, when not set for branch */
6109 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
6110
6111 /* exclude_kernel checked on syscall entry */
6112 if (!attr->exclude_kernel)
6113 mask |= PERF_SAMPLE_BRANCH_KERNEL;
6114
6115 if (!attr->exclude_user)
6116 mask |= PERF_SAMPLE_BRANCH_USER;
6117
6118 if (!attr->exclude_hv)
6119 mask |= PERF_SAMPLE_BRANCH_HV;
6120 /*
6121 * adjust user setting (for HW filter setup)
6122 */
6123 attr->branch_sample_type = mask;
6124 }
6125 }
6126out:
6127 return ret;
6128
6129err_size:
6130 put_user(sizeof(*attr), &uattr->size);
6131 ret = -E2BIG;
6132 goto out;
6133}
6134
6135static int
6136perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
6137{
6138 struct ring_buffer *rb = NULL, *old_rb = NULL;
6139 int ret = -EINVAL;
6140
6141 if (!output_event)
6142 goto set;
6143
6144 /* don't allow circular references */
6145 if (event == output_event)
6146 goto out;
6147
6148 /*
6149 * Don't allow cross-cpu buffers
6150 */
6151 if (output_event->cpu != event->cpu)
6152 goto out;
6153
6154 /*
6155 * If its not a per-cpu rb, it must be the same task.
6156 */
6157 if (output_event->cpu == -1 && output_event->ctx != event->ctx)
6158 goto out;
6159
6160set:
6161 mutex_lock(&event->mmap_mutex);
6162 /* Can't redirect output if we've got an active mmap() */
6163 if (atomic_read(&event->mmap_count))
6164 goto unlock;
6165
6166 if (output_event) {
6167 /* get the rb we want to redirect to */
6168 rb = ring_buffer_get(output_event);
6169 if (!rb)
6170 goto unlock;
6171 }
6172
6173 old_rb = event->rb;
6174 rcu_assign_pointer(event->rb, rb);
6175 if (old_rb)
6176 ring_buffer_detach(event, old_rb);
6177 ret = 0;
6178unlock:
6179 mutex_unlock(&event->mmap_mutex);
6180
6181 if (old_rb)
6182 ring_buffer_put(old_rb);
6183out:
6184 return ret;
6185}
6186
6187/**
6188 * sys_perf_event_open - open a performance event, associate it to a task/cpu
6189 *
6190 * @attr_uptr: event_id type attributes for monitoring/sampling
6191 * @pid: target pid
6192 * @cpu: target cpu
6193 * @group_fd: group leader event fd
6194 */
6195SYSCALL_DEFINE5(perf_event_open,
6196 struct perf_event_attr __user *, attr_uptr,
6197 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
6198{
6199 struct perf_event *group_leader = NULL, *output_event = NULL;
6200 struct perf_event *event, *sibling;
6201 struct perf_event_attr attr;
6202 struct perf_event_context *ctx;
6203 struct file *event_file = NULL;
6204 struct file *group_file = NULL;
6205 struct task_struct *task = NULL;
6206 struct pmu *pmu;
6207 int event_fd;
6208 int move_group = 0;
6209 int fput_needed = 0;
6210 int err;
6211
6212 /* for future expandability... */
6213 if (flags & ~PERF_FLAG_ALL)
6214 return -EINVAL;
6215
6216 err = perf_copy_attr(attr_uptr, &attr);
6217 if (err)
6218 return err;
6219
6220 if (!attr.exclude_kernel) {
6221 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6222 return -EACCES;
6223 }
6224
6225 if (attr.freq) {
6226 if (attr.sample_freq > sysctl_perf_event_sample_rate)
6227 return -EINVAL;
6228 }
6229
6230 /*
6231 * In cgroup mode, the pid argument is used to pass the fd
6232 * opened to the cgroup directory in cgroupfs. The cpu argument
6233 * designates the cpu on which to monitor threads from that
6234 * cgroup.
6235 */
6236 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
6237 return -EINVAL;
6238
6239 event_fd = get_unused_fd_flags(O_RDWR);
6240 if (event_fd < 0)
6241 return event_fd;
6242
6243 if (group_fd != -1) {
6244 group_file = perf_fget_light(group_fd, &fput_needed);
6245 if (IS_ERR(group_file)) {
6246 err = PTR_ERR(group_file);
6247 goto err_fd;
6248 }
6249 group_leader = group_file->private_data;
6250 if (flags & PERF_FLAG_FD_OUTPUT)
6251 output_event = group_leader;
6252 if (flags & PERF_FLAG_FD_NO_GROUP)
6253 group_leader = NULL;
6254 }
6255
6256 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
6257 task = find_lively_task_by_vpid(pid);
6258 if (IS_ERR(task)) {
6259 err = PTR_ERR(task);
6260 goto err_group_fd;
6261 }
6262 }
6263
6264 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
6265 NULL, NULL);
6266 if (IS_ERR(event)) {
6267 err = PTR_ERR(event);
6268 goto err_task;
6269 }
6270
6271 if (flags & PERF_FLAG_PID_CGROUP) {
6272 err = perf_cgroup_connect(pid, event, &attr, group_leader);
6273 if (err)
6274 goto err_alloc;
6275 /*
6276 * one more event:
6277 * - that has cgroup constraint on event->cpu
6278 * - that may need work on context switch
6279 */
6280 atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
6281 static_key_slow_inc(&perf_sched_events.key);
6282 }
6283
6284 /*
6285 * Special case software events and allow them to be part of
6286 * any hardware group.
6287 */
6288 pmu = event->pmu;
6289
6290 if (group_leader &&
6291 (is_software_event(event) != is_software_event(group_leader))) {
6292 if (is_software_event(event)) {
6293 /*
6294 * If event and group_leader are not both a software
6295 * event, and event is, then group leader is not.
6296 *
6297 * Allow the addition of software events to !software
6298 * groups, this is safe because software events never
6299 * fail to schedule.
6300 */
6301 pmu = group_leader->pmu;
6302 } else if (is_software_event(group_leader) &&
6303 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
6304 /*
6305 * In case the group is a pure software group, and we
6306 * try to add a hardware event, move the whole group to
6307 * the hardware context.
6308 */
6309 move_group = 1;
6310 }
6311 }
6312
6313 /*
6314 * Get the target context (task or percpu):
6315 */
6316 ctx = find_get_context(pmu, task, cpu);
6317 if (IS_ERR(ctx)) {
6318 err = PTR_ERR(ctx);
6319 goto err_alloc;
6320 }
6321
6322 if (task) {
6323 put_task_struct(task);
6324 task = NULL;
6325 }
6326
6327 /*
6328 * Look up the group leader (we will attach this event to it):
6329 */
6330 if (group_leader) {
6331 err = -EINVAL;
6332
6333 /*
6334 * Do not allow a recursive hierarchy (this new sibling
6335 * becoming part of another group-sibling):
6336 */
6337 if (group_leader->group_leader != group_leader)
6338 goto err_context;
6339 /*
6340 * Do not allow to attach to a group in a different
6341 * task or CPU context:
6342 */
6343 if (move_group) {
6344 if (group_leader->ctx->type != ctx->type)
6345 goto err_context;
6346 } else {
6347 if (group_leader->ctx != ctx)
6348 goto err_context;
6349 }
6350
6351 /*
6352 * Only a group leader can be exclusive or pinned
6353 */
6354 if (attr.exclusive || attr.pinned)
6355 goto err_context;
6356 }
6357
6358 if (output_event) {
6359 err = perf_event_set_output(event, output_event);
6360 if (err)
6361 goto err_context;
6362 }
6363
6364 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
6365 if (IS_ERR(event_file)) {
6366 err = PTR_ERR(event_file);
6367 goto err_context;
6368 }
6369
6370 if (move_group) {
6371 struct perf_event_context *gctx = group_leader->ctx;
6372
6373 mutex_lock(&gctx->mutex);
6374 perf_remove_from_context(group_leader);
6375 list_for_each_entry(sibling, &group_leader->sibling_list,
6376 group_entry) {
6377 perf_remove_from_context(sibling);
6378 put_ctx(gctx);
6379 }
6380 mutex_unlock(&gctx->mutex);
6381 put_ctx(gctx);
6382 }
6383
6384 WARN_ON_ONCE(ctx->parent_ctx);
6385 mutex_lock(&ctx->mutex);
6386
6387 if (move_group) {
6388 perf_install_in_context(ctx, group_leader, cpu);
6389 get_ctx(ctx);
6390 list_for_each_entry(sibling, &group_leader->sibling_list,
6391 group_entry) {
6392 perf_install_in_context(ctx, sibling, cpu);
6393 get_ctx(ctx);
6394 }
6395 }
6396
6397 perf_install_in_context(ctx, event, cpu);
6398 ++ctx->generation;
6399 perf_unpin_context(ctx);
6400 mutex_unlock(&ctx->mutex);
6401
6402 event->owner = current;
6403
6404 mutex_lock(¤t->perf_event_mutex);
6405 list_add_tail(&event->owner_entry, ¤t->perf_event_list);
6406 mutex_unlock(¤t->perf_event_mutex);
6407
6408 /*
6409 * Precalculate sample_data sizes
6410 */
6411 perf_event__header_size(event);
6412 perf_event__id_header_size(event);
6413
6414 /*
6415 * Drop the reference on the group_event after placing the
6416 * new event on the sibling_list. This ensures destruction
6417 * of the group leader will find the pointer to itself in
6418 * perf_group_detach().
6419 */
6420 fput_light(group_file, fput_needed);
6421 fd_install(event_fd, event_file);
6422 return event_fd;
6423
6424err_context:
6425 perf_unpin_context(ctx);
6426 put_ctx(ctx);
6427err_alloc:
6428 free_event(event);
6429err_task:
6430 if (task)
6431 put_task_struct(task);
6432err_group_fd:
6433 fput_light(group_file, fput_needed);
6434err_fd:
6435 put_unused_fd(event_fd);
6436 return err;
6437}
6438
6439/**
6440 * perf_event_create_kernel_counter
6441 *
6442 * @attr: attributes of the counter to create
6443 * @cpu: cpu in which the counter is bound
6444 * @task: task to profile (NULL for percpu)
6445 */
6446struct perf_event *
6447perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
6448 struct task_struct *task,
6449 perf_overflow_handler_t overflow_handler,
6450 void *context)
6451{
6452 struct perf_event_context *ctx;
6453 struct perf_event *event;
6454 int err;
6455
6456 /*
6457 * Get the target context (task or percpu):
6458 */
6459
6460 event = perf_event_alloc(attr, cpu, task, NULL, NULL,
6461 overflow_handler, context);
6462 if (IS_ERR(event)) {
6463 err = PTR_ERR(event);
6464 goto err;
6465 }
6466
6467 ctx = find_get_context(event->pmu, task, cpu);
6468 if (IS_ERR(ctx)) {
6469 err = PTR_ERR(ctx);
6470 goto err_free;
6471 }
6472
6473 WARN_ON_ONCE(ctx->parent_ctx);
6474 mutex_lock(&ctx->mutex);
6475 perf_install_in_context(ctx, event, cpu);
6476 ++ctx->generation;
6477 perf_unpin_context(ctx);
6478 mutex_unlock(&ctx->mutex);
6479
6480 return event;
6481
6482err_free:
6483 free_event(event);
6484err:
6485 return ERR_PTR(err);
6486}
6487EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6488
6489static void sync_child_event(struct perf_event *child_event,
6490 struct task_struct *child)
6491{
6492 struct perf_event *parent_event = child_event->parent;
6493 u64 child_val;
6494
6495 if (child_event->attr.inherit_stat)
6496 perf_event_read_event(child_event, child);
6497
6498 child_val = perf_event_count(child_event);
6499
6500 /*
6501 * Add back the child's count to the parent's count:
6502 */
6503 atomic64_add(child_val, &parent_event->child_count);
6504 atomic64_add(child_event->total_time_enabled,
6505 &parent_event->child_total_time_enabled);
6506 atomic64_add(child_event->total_time_running,
6507 &parent_event->child_total_time_running);
6508
6509 /*
6510 * Remove this event from the parent's list
6511 */
6512 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6513 mutex_lock(&parent_event->child_mutex);
6514 list_del_init(&child_event->child_list);
6515 mutex_unlock(&parent_event->child_mutex);
6516
6517 /*
6518 * Release the parent event, if this was the last
6519 * reference to it.
6520 */
6521 put_event(parent_event);
6522}
6523
6524static void
6525__perf_event_exit_task(struct perf_event *child_event,
6526 struct perf_event_context *child_ctx,
6527 struct task_struct *child)
6528{
6529 if (child_event->parent) {
6530 raw_spin_lock_irq(&child_ctx->lock);
6531 perf_group_detach(child_event);
6532 raw_spin_unlock_irq(&child_ctx->lock);
6533 }
6534
6535 perf_remove_from_context(child_event);
6536
6537 /*
6538 * It can happen that the parent exits first, and has events
6539 * that are still around due to the child reference. These
6540 * events need to be zapped.
6541 */
6542 if (child_event->parent) {
6543 sync_child_event(child_event, child);
6544 free_event(child_event);
6545 }
6546}
6547
6548static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6549{
6550 struct perf_event *child_event, *tmp;
6551 struct perf_event_context *child_ctx;
6552 unsigned long flags;
6553
6554 if (likely(!child->perf_event_ctxp[ctxn])) {
6555 perf_event_task(child, NULL, 0);
6556 return;
6557 }
6558
6559 local_irq_save(flags);
6560 /*
6561 * We can't reschedule here because interrupts are disabled,
6562 * and either child is current or it is a task that can't be
6563 * scheduled, so we are now safe from rescheduling changing
6564 * our context.
6565 */
6566 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6567
6568 /*
6569 * Take the context lock here so that if find_get_context is
6570 * reading child->perf_event_ctxp, we wait until it has
6571 * incremented the context's refcount before we do put_ctx below.
6572 */
6573 raw_spin_lock(&child_ctx->lock);
6574 task_ctx_sched_out(child_ctx);
6575 child->perf_event_ctxp[ctxn] = NULL;
6576 /*
6577 * If this context is a clone; unclone it so it can't get
6578 * swapped to another process while we're removing all
6579 * the events from it.
6580 */
6581 unclone_ctx(child_ctx);
6582 update_context_time(child_ctx);
6583 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6584
6585 /*
6586 * Report the task dead after unscheduling the events so that we
6587 * won't get any samples after PERF_RECORD_EXIT. We can however still
6588 * get a few PERF_RECORD_READ events.
6589 */
6590 perf_event_task(child, child_ctx, 0);
6591
6592 /*
6593 * We can recurse on the same lock type through:
6594 *
6595 * __perf_event_exit_task()
6596 * sync_child_event()
6597 * put_event()
6598 * mutex_lock(&ctx->mutex)
6599 *
6600 * But since its the parent context it won't be the same instance.
6601 */
6602 mutex_lock(&child_ctx->mutex);
6603
6604again:
6605 list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
6606 group_entry)
6607 __perf_event_exit_task(child_event, child_ctx, child);
6608
6609 list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6610 group_entry)
6611 __perf_event_exit_task(child_event, child_ctx, child);
6612
6613 /*
6614 * If the last event was a group event, it will have appended all
6615 * its siblings to the list, but we obtained 'tmp' before that which
6616 * will still point to the list head terminating the iteration.
6617 */
6618 if (!list_empty(&child_ctx->pinned_groups) ||
6619 !list_empty(&child_ctx->flexible_groups))
6620 goto again;
6621
6622 mutex_unlock(&child_ctx->mutex);
6623
6624 put_ctx(child_ctx);
6625}
6626
6627/*
6628 * When a child task exits, feed back event values to parent events.
6629 */
6630void perf_event_exit_task(struct task_struct *child)
6631{
6632 struct perf_event *event, *tmp;
6633 int ctxn;
6634
6635 mutex_lock(&child->perf_event_mutex);
6636 list_for_each_entry_safe(event, tmp, &child->perf_event_list,
6637 owner_entry) {
6638 list_del_init(&event->owner_entry);
6639
6640 /*
6641 * Ensure the list deletion is visible before we clear
6642 * the owner, closes a race against perf_release() where
6643 * we need to serialize on the owner->perf_event_mutex.
6644 */
6645 smp_wmb();
6646 event->owner = NULL;
6647 }
6648 mutex_unlock(&child->perf_event_mutex);
6649
6650 for_each_task_context_nr(ctxn)
6651 perf_event_exit_task_context(child, ctxn);
6652}
6653
6654static void perf_free_event(struct perf_event *event,
6655 struct perf_event_context *ctx)
6656{
6657 struct perf_event *parent = event->parent;
6658
6659 if (WARN_ON_ONCE(!parent))
6660 return;
6661
6662 mutex_lock(&parent->child_mutex);
6663 list_del_init(&event->child_list);
6664 mutex_unlock(&parent->child_mutex);
6665
6666 put_event(parent);
6667
6668 perf_group_detach(event);
6669 list_del_event(event, ctx);
6670 free_event(event);
6671}
6672
6673/*
6674 * free an unexposed, unused context as created by inheritance by
6675 * perf_event_init_task below, used by fork() in case of fail.
6676 */
6677void perf_event_free_task(struct task_struct *task)
6678{
6679 struct perf_event_context *ctx;
6680 struct perf_event *event, *tmp;
6681 int ctxn;
6682
6683 for_each_task_context_nr(ctxn) {
6684 ctx = task->perf_event_ctxp[ctxn];
6685 if (!ctx)
6686 continue;
6687
6688 mutex_lock(&ctx->mutex);
6689again:
6690 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
6691 group_entry)
6692 perf_free_event(event, ctx);
6693
6694 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
6695 group_entry)
6696 perf_free_event(event, ctx);
6697
6698 if (!list_empty(&ctx->pinned_groups) ||
6699 !list_empty(&ctx->flexible_groups))
6700 goto again;
6701
6702 mutex_unlock(&ctx->mutex);
6703
6704 put_ctx(ctx);
6705 }
6706}
6707
6708void perf_event_delayed_put(struct task_struct *task)
6709{
6710 int ctxn;
6711
6712 for_each_task_context_nr(ctxn)
6713 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
6714}
6715
6716/*
6717 * inherit a event from parent task to child task:
6718 */
6719static struct perf_event *
6720inherit_event(struct perf_event *parent_event,
6721 struct task_struct *parent,
6722 struct perf_event_context *parent_ctx,
6723 struct task_struct *child,
6724 struct perf_event *group_leader,
6725 struct perf_event_context *child_ctx)
6726{
6727 struct perf_event *child_event;
6728 unsigned long flags;
6729
6730 /*
6731 * Instead of creating recursive hierarchies of events,
6732 * we link inherited events back to the original parent,
6733 * which has a filp for sure, which we use as the reference
6734 * count:
6735 */
6736 if (parent_event->parent)
6737 parent_event = parent_event->parent;
6738
6739 child_event = perf_event_alloc(&parent_event->attr,
6740 parent_event->cpu,
6741 child,
6742 group_leader, parent_event,
6743 NULL, NULL);
6744 if (IS_ERR(child_event))
6745 return child_event;
6746
6747 if (!atomic_long_inc_not_zero(&parent_event->refcount)) {
6748 free_event(child_event);
6749 return NULL;
6750 }
6751
6752 get_ctx(child_ctx);
6753
6754 /*
6755 * Make the child state follow the state of the parent event,
6756 * not its attr.disabled bit. We hold the parent's mutex,
6757 * so we won't race with perf_event_{en, dis}able_family.
6758 */
6759 if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
6760 child_event->state = PERF_EVENT_STATE_INACTIVE;
6761 else
6762 child_event->state = PERF_EVENT_STATE_OFF;
6763
6764 if (parent_event->attr.freq) {
6765 u64 sample_period = parent_event->hw.sample_period;
6766 struct hw_perf_event *hwc = &child_event->hw;
6767
6768 hwc->sample_period = sample_period;
6769 hwc->last_period = sample_period;
6770
6771 local64_set(&hwc->period_left, sample_period);
6772 }
6773
6774 child_event->ctx = child_ctx;
6775 child_event->overflow_handler = parent_event->overflow_handler;
6776 child_event->overflow_handler_context
6777 = parent_event->overflow_handler_context;
6778
6779 /*
6780 * Precalculate sample_data sizes
6781 */
6782 perf_event__header_size(child_event);
6783 perf_event__id_header_size(child_event);
6784
6785 /*
6786 * Link it up in the child's context:
6787 */
6788 raw_spin_lock_irqsave(&child_ctx->lock, flags);
6789 add_event_to_ctx(child_event, child_ctx);
6790 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6791
6792 /*
6793 * Link this into the parent event's child list
6794 */
6795 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6796 mutex_lock(&parent_event->child_mutex);
6797 list_add_tail(&child_event->child_list, &parent_event->child_list);
6798 mutex_unlock(&parent_event->child_mutex);
6799
6800 return child_event;
6801}
6802
6803static int inherit_group(struct perf_event *parent_event,
6804 struct task_struct *parent,
6805 struct perf_event_context *parent_ctx,
6806 struct task_struct *child,
6807 struct perf_event_context *child_ctx)
6808{
6809 struct perf_event *leader;
6810 struct perf_event *sub;
6811 struct perf_event *child_ctr;
6812
6813 leader = inherit_event(parent_event, parent, parent_ctx,
6814 child, NULL, child_ctx);
6815 if (IS_ERR(leader))
6816 return PTR_ERR(leader);
6817 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
6818 child_ctr = inherit_event(sub, parent, parent_ctx,
6819 child, leader, child_ctx);
6820 if (IS_ERR(child_ctr))
6821 return PTR_ERR(child_ctr);
6822 }
6823 return 0;
6824}
6825
6826static int
6827inherit_task_group(struct perf_event *event, struct task_struct *parent,
6828 struct perf_event_context *parent_ctx,
6829 struct task_struct *child, int ctxn,
6830 int *inherited_all)
6831{
6832 int ret;
6833 struct perf_event_context *child_ctx;
6834
6835 if (!event->attr.inherit) {
6836 *inherited_all = 0;
6837 return 0;
6838 }
6839
6840 child_ctx = child->perf_event_ctxp[ctxn];
6841 if (!child_ctx) {
6842 /*
6843 * This is executed from the parent task context, so
6844 * inherit events that have been marked for cloning.
6845 * First allocate and initialize a context for the
6846 * child.
6847 */
6848
6849 child_ctx = alloc_perf_context(event->pmu, child);
6850 if (!child_ctx)
6851 return -ENOMEM;
6852
6853 child->perf_event_ctxp[ctxn] = child_ctx;
6854 }
6855
6856 ret = inherit_group(event, parent, parent_ctx,
6857 child, child_ctx);
6858
6859 if (ret)
6860 *inherited_all = 0;
6861
6862 return ret;
6863}
6864
6865/*
6866 * Initialize the perf_event context in task_struct
6867 */
6868int perf_event_init_context(struct task_struct *child, int ctxn)
6869{
6870 struct perf_event_context *child_ctx, *parent_ctx;
6871 struct perf_event_context *cloned_ctx;
6872 struct perf_event *event;
6873 struct task_struct *parent = current;
6874 int inherited_all = 1;
6875 unsigned long flags;
6876 int ret = 0;
6877
6878 if (likely(!parent->perf_event_ctxp[ctxn]))
6879 return 0;
6880
6881 /*
6882 * If the parent's context is a clone, pin it so it won't get
6883 * swapped under us.
6884 */
6885 parent_ctx = perf_pin_task_context(parent, ctxn);
6886
6887 /*
6888 * No need to check if parent_ctx != NULL here; since we saw
6889 * it non-NULL earlier, the only reason for it to become NULL
6890 * is if we exit, and since we're currently in the middle of
6891 * a fork we can't be exiting at the same time.
6892 */
6893
6894 /*
6895 * Lock the parent list. No need to lock the child - not PID
6896 * hashed yet and not running, so nobody can access it.
6897 */
6898 mutex_lock(&parent_ctx->mutex);
6899
6900 /*
6901 * We dont have to disable NMIs - we are only looking at
6902 * the list, not manipulating it:
6903 */
6904 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
6905 ret = inherit_task_group(event, parent, parent_ctx,
6906 child, ctxn, &inherited_all);
6907 if (ret)
6908 break;
6909 }
6910
6911 /*
6912 * We can't hold ctx->lock when iterating the ->flexible_group list due
6913 * to allocations, but we need to prevent rotation because
6914 * rotate_ctx() will change the list from interrupt context.
6915 */
6916 raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6917 parent_ctx->rotate_disable = 1;
6918 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6919
6920 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
6921 ret = inherit_task_group(event, parent, parent_ctx,
6922 child, ctxn, &inherited_all);
6923 if (ret)
6924 break;
6925 }
6926
6927 raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6928 parent_ctx->rotate_disable = 0;
6929
6930 child_ctx = child->perf_event_ctxp[ctxn];
6931
6932 if (child_ctx && inherited_all) {
6933 /*
6934 * Mark the child context as a clone of the parent
6935 * context, or of whatever the parent is a clone of.
6936 *
6937 * Note that if the parent is a clone, the holding of
6938 * parent_ctx->lock avoids it from being uncloned.
6939 */
6940 cloned_ctx = parent_ctx->parent_ctx;
6941 if (cloned_ctx) {
6942 child_ctx->parent_ctx = cloned_ctx;
6943 child_ctx->parent_gen = parent_ctx->parent_gen;
6944 } else {
6945 child_ctx->parent_ctx = parent_ctx;
6946 child_ctx->parent_gen = parent_ctx->generation;
6947 }
6948 get_ctx(child_ctx->parent_ctx);
6949 }
6950
6951 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6952 mutex_unlock(&parent_ctx->mutex);
6953
6954 perf_unpin_context(parent_ctx);
6955 put_ctx(parent_ctx);
6956
6957 return ret;
6958}
6959
6960/*
6961 * Initialize the perf_event context in task_struct
6962 */
6963int perf_event_init_task(struct task_struct *child)
6964{
6965 int ctxn, ret;
6966
6967 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
6968 mutex_init(&child->perf_event_mutex);
6969 INIT_LIST_HEAD(&child->perf_event_list);
6970
6971 for_each_task_context_nr(ctxn) {
6972 ret = perf_event_init_context(child, ctxn);
6973 if (ret)
6974 return ret;
6975 }
6976
6977 return 0;
6978}
6979
6980static void __init perf_event_init_all_cpus(void)
6981{
6982 struct swevent_htable *swhash;
6983 int cpu;
6984
6985 for_each_possible_cpu(cpu) {
6986 swhash = &per_cpu(swevent_htable, cpu);
6987 mutex_init(&swhash->hlist_mutex);
6988 INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
6989 }
6990}
6991
6992static void __cpuinit perf_event_init_cpu(int cpu)
6993{
6994 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6995
6996 mutex_lock(&swhash->hlist_mutex);
6997 if (swhash->hlist_refcount > 0) {
6998 struct swevent_hlist *hlist;
6999
7000 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
7001 WARN_ON(!hlist);
7002 rcu_assign_pointer(swhash->swevent_hlist, hlist);
7003 }
7004 mutex_unlock(&swhash->hlist_mutex);
7005}
7006
7007#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
7008static void perf_pmu_rotate_stop(struct pmu *pmu)
7009{
7010 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
7011
7012 WARN_ON(!irqs_disabled());
7013
7014 list_del_init(&cpuctx->rotation_list);
7015}
7016
7017static void __perf_event_exit_context(void *__info)
7018{
7019 struct perf_event_context *ctx = __info;
7020 struct perf_event *event, *tmp;
7021
7022 perf_pmu_rotate_stop(ctx->pmu);
7023
7024 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
7025 __perf_remove_from_context(event);
7026 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
7027 __perf_remove_from_context(event);
7028}
7029
7030static void perf_event_exit_cpu_context(int cpu)
7031{
7032 struct perf_event_context *ctx;
7033 struct pmu *pmu;
7034 int idx;
7035
7036 idx = srcu_read_lock(&pmus_srcu);
7037 list_for_each_entry_rcu(pmu, &pmus, entry) {
7038 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
7039
7040 mutex_lock(&ctx->mutex);
7041 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
7042 mutex_unlock(&ctx->mutex);
7043 }
7044 srcu_read_unlock(&pmus_srcu, idx);
7045}
7046
7047static void perf_event_exit_cpu(int cpu)
7048{
7049 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
7050
7051 mutex_lock(&swhash->hlist_mutex);
7052 swevent_hlist_release(swhash);
7053 mutex_unlock(&swhash->hlist_mutex);
7054
7055 perf_event_exit_cpu_context(cpu);
7056}
7057#else
7058static inline void perf_event_exit_cpu(int cpu) { }
7059#endif
7060
7061static int
7062perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
7063{
7064 int cpu;
7065
7066 for_each_online_cpu(cpu)
7067 perf_event_exit_cpu(cpu);
7068
7069 return NOTIFY_OK;
7070}
7071
7072/*
7073 * Run the perf reboot notifier at the very last possible moment so that
7074 * the generic watchdog code runs as long as possible.
7075 */
7076static struct notifier_block perf_reboot_notifier = {
7077 .notifier_call = perf_reboot,
7078 .priority = INT_MIN,
7079};
7080
7081static int __cpuinit
7082perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
7083{
7084 unsigned int cpu = (long)hcpu;
7085
7086 switch (action & ~CPU_TASKS_FROZEN) {
7087
7088 case CPU_UP_PREPARE:
7089 case CPU_DOWN_FAILED:
7090 perf_event_init_cpu(cpu);
7091 break;
7092
7093 case CPU_UP_CANCELED:
7094 case CPU_DOWN_PREPARE:
7095 perf_event_exit_cpu(cpu);
7096 break;
7097
7098 default:
7099 break;
7100 }
7101
7102 return NOTIFY_OK;
7103}
7104
7105void __init perf_event_init(void)
7106{
7107 int ret;
7108
7109 idr_init(&pmu_idr);
7110
7111 perf_event_init_all_cpus();
7112 init_srcu_struct(&pmus_srcu);
7113 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
7114 perf_pmu_register(&perf_cpu_clock, NULL, -1);
7115 perf_pmu_register(&perf_task_clock, NULL, -1);
7116 perf_tp_register();
7117 perf_cpu_notifier(perf_cpu_notify);
7118 register_reboot_notifier(&perf_reboot_notifier);
7119
7120 ret = init_hw_breakpoint();
7121 WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
7122
7123 /* do not patch jump label more than once per second */
7124 jump_label_rate_limit(&perf_sched_events, HZ);
7125
7126 /*
7127 * Build time assertion that we keep the data_head at the intended
7128 * location. IOW, validation we got the __reserved[] size right.
7129 */
7130 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
7131 != 1024);
7132}
7133
7134static int __init perf_event_sysfs_init(void)
7135{
7136 struct pmu *pmu;
7137 int ret;
7138
7139 mutex_lock(&pmus_lock);
7140
7141 ret = bus_register(&pmu_bus);
7142 if (ret)
7143 goto unlock;
7144
7145 list_for_each_entry(pmu, &pmus, entry) {
7146 if (!pmu->name || pmu->type < 0)
7147 continue;
7148
7149 ret = pmu_dev_alloc(pmu);
7150 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
7151 }
7152 pmu_bus_running = 1;
7153 ret = 0;
7154
7155unlock:
7156 mutex_unlock(&pmus_lock);
7157
7158 return ret;
7159}
7160device_initcall(perf_event_sysfs_init);
7161
7162#ifdef CONFIG_CGROUP_PERF
7163static struct cgroup_subsys_state *perf_cgroup_create(struct cgroup *cont)
7164{
7165 struct perf_cgroup *jc;
7166
7167 jc = kzalloc(sizeof(*jc), GFP_KERNEL);
7168 if (!jc)
7169 return ERR_PTR(-ENOMEM);
7170
7171 jc->info = alloc_percpu(struct perf_cgroup_info);
7172 if (!jc->info) {
7173 kfree(jc);
7174 return ERR_PTR(-ENOMEM);
7175 }
7176
7177 return &jc->css;
7178}
7179
7180static void perf_cgroup_destroy(struct cgroup *cont)
7181{
7182 struct perf_cgroup *jc;
7183 jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
7184 struct perf_cgroup, css);
7185 free_percpu(jc->info);
7186 kfree(jc);
7187}
7188
7189static int __perf_cgroup_move(void *info)
7190{
7191 struct task_struct *task = info;
7192 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
7193 return 0;
7194}
7195
7196static void perf_cgroup_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
7197{
7198 struct task_struct *task;
7199
7200 cgroup_taskset_for_each(task, cgrp, tset)
7201 task_function_call(task, __perf_cgroup_move, task);
7202}
7203
7204static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
7205 struct task_struct *task)
7206{
7207 /*
7208 * cgroup_exit() is called in the copy_process() failure path.
7209 * Ignore this case since the task hasn't ran yet, this avoids
7210 * trying to poke a half freed task state from generic code.
7211 */
7212 if (!(task->flags & PF_EXITING))
7213 return;
7214
7215 task_function_call(task, __perf_cgroup_move, task);
7216}
7217
7218struct cgroup_subsys perf_subsys = {
7219 .name = "perf_event",
7220 .subsys_id = perf_subsys_id,
7221 .create = perf_cgroup_create,
7222 .destroy = perf_cgroup_destroy,
7223 .exit = perf_cgroup_exit,
7224 .attach = perf_cgroup_attach,
7225};
7226#endif /* CONFIG_CGROUP_PERF */