1/*
  2 * Performance events callchain code, extracted from core.c:
  3 *
  4 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  6 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
  7 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  8 *
  9 * For licensing details see kernel-base/COPYING
 10 */
 11
 12#include <linux/perf_event.h>
 13#include <linux/slab.h>
 
 
 14#include "internal.h"
 15
 16struct callchain_cpus_entries {
 17	struct rcu_head			rcu_head;
 18	struct perf_callchain_entry	*cpu_entries[0];
 19};
 20
 21int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
 22int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
 23
 24static inline size_t perf_callchain_entry__sizeof(void)
 25{
 26	return (sizeof(struct perf_callchain_entry) +
 27		sizeof(__u64) * (sysctl_perf_event_max_stack +
 28				 sysctl_perf_event_max_contexts_per_stack));
 29}
 30
 31static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
 32static atomic_t nr_callchain_events;
 33static DEFINE_MUTEX(callchain_mutex);
 34static struct callchain_cpus_entries *callchain_cpus_entries;
 35
 36
 37__weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
 38				  struct pt_regs *regs)
 39{
 40}
 41
 42__weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
 43				struct pt_regs *regs)
 44{
 45}
 46
 47static void release_callchain_buffers_rcu(struct rcu_head *head)
 48{
 49	struct callchain_cpus_entries *entries;
 50	int cpu;
 51
 52	entries = container_of(head, struct callchain_cpus_entries, rcu_head);
 53
 54	for_each_possible_cpu(cpu)
 55		kfree(entries->cpu_entries[cpu]);
 56
 57	kfree(entries);
 58}
 59
 60static void release_callchain_buffers(void)
 61{
 62	struct callchain_cpus_entries *entries;
 63
 64	entries = callchain_cpus_entries;
 65	RCU_INIT_POINTER(callchain_cpus_entries, NULL);
 66	call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
 67}
 68
 69static int alloc_callchain_buffers(void)
 70{
 71	int cpu;
 72	int size;
 73	struct callchain_cpus_entries *entries;
 74
 75	/*
 76	 * We can't use the percpu allocation API for data that can be
 77	 * accessed from NMI. Use a temporary manual per cpu allocation
 78	 * until that gets sorted out.
 79	 */
 80	size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
 81
 82	entries = kzalloc(size, GFP_KERNEL);
 83	if (!entries)
 84		return -ENOMEM;
 85
 86	size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
 87
 88	for_each_possible_cpu(cpu) {
 89		entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
 90							 cpu_to_node(cpu));
 91		if (!entries->cpu_entries[cpu])
 92			goto fail;
 93	}
 94
 95	rcu_assign_pointer(callchain_cpus_entries, entries);
 96
 97	return 0;
 98
 99fail:
100	for_each_possible_cpu(cpu)
101		kfree(entries->cpu_entries[cpu]);
102	kfree(entries);
103
104	return -ENOMEM;
105}
106
107int get_callchain_buffers(int event_max_stack)
108{
109	int err = 0;
110	int count;
111
112	mutex_lock(&callchain_mutex);
113
114	count = atomic_inc_return(&nr_callchain_events);
115	if (WARN_ON_ONCE(count < 1)) {
116		err = -EINVAL;
117		goto exit;
118	}
119
120	if (count > 1) {
121		/* If the allocation failed, give up */
122		if (!callchain_cpus_entries)
123			err = -ENOMEM;
124		/*
125		 * If requesting per event more than the global cap,
126		 * return a different error to help userspace figure
127		 * this out.
128		 *
129		 * And also do it here so that we have &callchain_mutex held.
130		 */
131		if (event_max_stack > sysctl_perf_event_max_stack)
132			err = -EOVERFLOW;
133		goto exit;
134	}
135
136	err = alloc_callchain_buffers();
 
137exit:
138	if (err)
139		atomic_dec(&nr_callchain_events);
140
141	mutex_unlock(&callchain_mutex);
142
143	return err;
144}
145
146void put_callchain_buffers(void)
147{
148	if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
149		release_callchain_buffers();
150		mutex_unlock(&callchain_mutex);
151	}
152}
153
154static struct perf_callchain_entry *get_callchain_entry(int *rctx)
155{
156	int cpu;
157	struct callchain_cpus_entries *entries;
158
159	*rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
160	if (*rctx == -1)
161		return NULL;
162
163	entries = rcu_dereference(callchain_cpus_entries);
164	if (!entries)
 
165		return NULL;
 
166
167	cpu = smp_processor_id();
168
169	return (((void *)entries->cpu_entries[cpu]) +
170		(*rctx * perf_callchain_entry__sizeof()));
171}
172
173static void
174put_callchain_entry(int rctx)
175{
176	put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
177}
178
179struct perf_callchain_entry *
180perf_callchain(struct perf_event *event, struct pt_regs *regs)
181{
182	bool kernel = !event->attr.exclude_callchain_kernel;
183	bool user   = !event->attr.exclude_callchain_user;
184	/* Disallow cross-task user callchains. */
185	bool crosstask = event->ctx->task && event->ctx->task != current;
186	const u32 max_stack = event->attr.sample_max_stack;
187
188	if (!kernel && !user)
189		return NULL;
190
191	return get_perf_callchain(regs, 0, kernel, user, max_stack, crosstask, true);
192}
193
194struct perf_callchain_entry *
195get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
196		   u32 max_stack, bool crosstask, bool add_mark)
197{
198	struct perf_callchain_entry *entry;
199	struct perf_callchain_entry_ctx ctx;
200	int rctx;
201
202	entry = get_callchain_entry(&rctx);
203	if (rctx == -1)
204		return NULL;
205
206	if (!entry)
207		goto exit_put;
208
209	ctx.entry     = entry;
210	ctx.max_stack = max_stack;
211	ctx.nr	      = entry->nr = init_nr;
212	ctx.contexts       = 0;
213	ctx.contexts_maxed = false;
214
215	if (kernel && !user_mode(regs)) {
216		if (add_mark)
217			perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
218		perf_callchain_kernel(&ctx, regs);
219	}
220
221	if (user) {
222		if (!user_mode(regs)) {
223			if  (current->mm)
224				regs = task_pt_regs(current);
225			else
226				regs = NULL;
227		}
228
229		if (regs) {
230			if (crosstask)
231				goto exit_put;
232
233			if (add_mark)
234				perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
 
235			perf_callchain_user(&ctx, regs);
236		}
237	}
238
239exit_put:
240	put_callchain_entry(rctx);
241
242	return entry;
243}
244
245/*
246 * Used for sysctl_perf_event_max_stack and
247 * sysctl_perf_event_max_contexts_per_stack.
248 */
249int perf_event_max_stack_handler(struct ctl_table *table, int write,
250				 void __user *buffer, size_t *lenp, loff_t *ppos)
251{
252	int *value = table->data;
253	int new_value = *value, ret;
254	struct ctl_table new_table = *table;
255
256	new_table.data = &new_value;
257	ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
258	if (ret || !write)
259		return ret;
260
261	mutex_lock(&callchain_mutex);
262	if (atomic_read(&nr_callchain_events))
263		ret = -EBUSY;
264	else
265		*value = new_value;
266
267	mutex_unlock(&callchain_mutex);
268
269	return ret;
270}

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Performance events callchain code, extracted from core.c:
  4 *
  5 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  6 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  7 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
  8 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 
 
  9 */
 10
 11#include <linux/perf_event.h>
 12#include <linux/slab.h>
 13#include <linux/sched/task_stack.h>
 14
 15#include "internal.h"
 16
 17struct callchain_cpus_entries {
 18	struct rcu_head			rcu_head;
 19	struct perf_callchain_entry	*cpu_entries[];
 20};
 21
 22int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
 23int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
 24
 25static inline size_t perf_callchain_entry__sizeof(void)
 26{
 27	return (sizeof(struct perf_callchain_entry) +
 28		sizeof(__u64) * (sysctl_perf_event_max_stack +
 29				 sysctl_perf_event_max_contexts_per_stack));
 30}
 31
 32static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
 33static atomic_t nr_callchain_events;
 34static DEFINE_MUTEX(callchain_mutex);
 35static struct callchain_cpus_entries *callchain_cpus_entries;
 36
 37
 38__weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
 39				  struct pt_regs *regs)
 40{
 41}
 42
 43__weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
 44				struct pt_regs *regs)
 45{
 46}
 47
 48static void release_callchain_buffers_rcu(struct rcu_head *head)
 49{
 50	struct callchain_cpus_entries *entries;
 51	int cpu;
 52
 53	entries = container_of(head, struct callchain_cpus_entries, rcu_head);
 54
 55	for_each_possible_cpu(cpu)
 56		kfree(entries->cpu_entries[cpu]);
 57
 58	kfree(entries);
 59}
 60
 61static void release_callchain_buffers(void)
 62{
 63	struct callchain_cpus_entries *entries;
 64
 65	entries = callchain_cpus_entries;
 66	RCU_INIT_POINTER(callchain_cpus_entries, NULL);
 67	call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
 68}
 69
 70static int alloc_callchain_buffers(void)
 71{
 72	int cpu;
 73	int size;
 74	struct callchain_cpus_entries *entries;
 75
 76	/*
 77	 * We can't use the percpu allocation API for data that can be
 78	 * accessed from NMI. Use a temporary manual per cpu allocation
 79	 * until that gets sorted out.
 80	 */
 81	size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
 82
 83	entries = kzalloc(size, GFP_KERNEL);
 84	if (!entries)
 85		return -ENOMEM;
 86
 87	size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
 88
 89	for_each_possible_cpu(cpu) {
 90		entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
 91							 cpu_to_node(cpu));
 92		if (!entries->cpu_entries[cpu])
 93			goto fail;
 94	}
 95
 96	rcu_assign_pointer(callchain_cpus_entries, entries);
 97
 98	return 0;
 99
100fail:
101	for_each_possible_cpu(cpu)
102		kfree(entries->cpu_entries[cpu]);
103	kfree(entries);
104
105	return -ENOMEM;
106}
107
108int get_callchain_buffers(int event_max_stack)
109{
110	int err = 0;
111	int count;
112
113	mutex_lock(&callchain_mutex);
114
115	count = atomic_inc_return(&nr_callchain_events);
116	if (WARN_ON_ONCE(count < 1)) {
117		err = -EINVAL;
118		goto exit;
119	}
120
121	/*
122	 * If requesting per event more than the global cap,
123	 * return a different error to help userspace figure
124	 * this out.
125	 *
126	 * And also do it here so that we have &callchain_mutex held.
127	 */
128	if (event_max_stack > sysctl_perf_event_max_stack) {
129		err = -EOVERFLOW;
 
 
 
 
130		goto exit;
131	}
132
133	if (count == 1)
134		err = alloc_callchain_buffers();
135exit:
136	if (err)
137		atomic_dec(&nr_callchain_events);
138
139	mutex_unlock(&callchain_mutex);
140
141	return err;
142}
143
144void put_callchain_buffers(void)
145{
146	if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
147		release_callchain_buffers();
148		mutex_unlock(&callchain_mutex);
149	}
150}
151
152struct perf_callchain_entry *get_callchain_entry(int *rctx)
153{
154	int cpu;
155	struct callchain_cpus_entries *entries;
156
157	*rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
158	if (*rctx == -1)
159		return NULL;
160
161	entries = rcu_dereference(callchain_cpus_entries);
162	if (!entries) {
163		put_recursion_context(this_cpu_ptr(callchain_recursion), *rctx);
164		return NULL;
165	}
166
167	cpu = smp_processor_id();
168
169	return (((void *)entries->cpu_entries[cpu]) +
170		(*rctx * perf_callchain_entry__sizeof()));
171}
172
173void
174put_callchain_entry(int rctx)
175{
176	put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
177}
178
179struct perf_callchain_entry *
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
181		   u32 max_stack, bool crosstask, bool add_mark)
182{
183	struct perf_callchain_entry *entry;
184	struct perf_callchain_entry_ctx ctx;
185	int rctx;
186
187	entry = get_callchain_entry(&rctx);
 
 
 
188	if (!entry)
189		return NULL;
190
191	ctx.entry     = entry;
192	ctx.max_stack = max_stack;
193	ctx.nr	      = entry->nr = init_nr;
194	ctx.contexts       = 0;
195	ctx.contexts_maxed = false;
196
197	if (kernel && !user_mode(regs)) {
198		if (add_mark)
199			perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
200		perf_callchain_kernel(&ctx, regs);
201	}
202
203	if (user) {
204		if (!user_mode(regs)) {
205			if  (current->mm)
206				regs = task_pt_regs(current);
207			else
208				regs = NULL;
209		}
210
211		if (regs) {
212			if (crosstask)
213				goto exit_put;
214
215			if (add_mark)
216				perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
217
218			perf_callchain_user(&ctx, regs);
219		}
220	}
221
222exit_put:
223	put_callchain_entry(rctx);
224
225	return entry;
226}
227
228/*
229 * Used for sysctl_perf_event_max_stack and
230 * sysctl_perf_event_max_contexts_per_stack.
231 */
232int perf_event_max_stack_handler(struct ctl_table *table, int write,
233				 void *buffer, size_t *lenp, loff_t *ppos)
234{
235	int *value = table->data;
236	int new_value = *value, ret;
237	struct ctl_table new_table = *table;
238
239	new_table.data = &new_value;
240	ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
241	if (ret || !write)
242		return ret;
243
244	mutex_lock(&callchain_mutex);
245	if (atomic_read(&nr_callchain_events))
246		ret = -EBUSY;
247	else
248		*value = new_value;
249
250	mutex_unlock(&callchain_mutex);
251
252	return ret;
253}