1/**
  2 * @file cpu_buffer.c
  3 *
  4 * @remark Copyright 2002-2009 OProfile authors
  5 * @remark Read the file COPYING
  6 *
  7 * @author John Levon <levon@movementarian.org>
  8 * @author Barry Kasindorf <barry.kasindorf@amd.com>
  9 * @author Robert Richter <robert.richter@amd.com>
 10 *
 11 * Each CPU has a local buffer that stores PC value/event
 12 * pairs. We also log context switches when we notice them.
 13 * Eventually each CPU's buffer is processed into the global
 14 * event buffer by sync_buffer().
 15 *
 16 * We use a local buffer for two reasons: an NMI or similar
 17 * interrupt cannot synchronise, and high sampling rates
 18 * would lead to catastrophic global synchronisation if
 19 * a global buffer was used.
 20 */
 21
 22#include <linux/sched.h>
 23#include <linux/oprofile.h>
 24#include <linux/errno.h>
 25
 26#include <asm/ptrace.h>
 27
 28#include "event_buffer.h"
 29#include "cpu_buffer.h"
 30#include "buffer_sync.h"
 31#include "oprof.h"
 32
 33#define OP_BUFFER_FLAGS	0
 34
 35static struct trace_buffer *op_ring_buffer;
 36DEFINE_PER_CPU(struct oprofile_cpu_buffer, op_cpu_buffer);
 37
 38static void wq_sync_buffer(struct work_struct *work);
 39
 40#define DEFAULT_TIMER_EXPIRE (HZ / 10)
 41static int work_enabled;
 42
 43unsigned long oprofile_get_cpu_buffer_size(void)
 44{
 45	return oprofile_cpu_buffer_size;
 46}
 47
 48void oprofile_cpu_buffer_inc_smpl_lost(void)
 49{
 50	struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer);
 51
 52	cpu_buf->sample_lost_overflow++;
 53}
 54
 55void free_cpu_buffers(void)
 56{
 57	if (op_ring_buffer)
 58		ring_buffer_free(op_ring_buffer);
 59	op_ring_buffer = NULL;
 60}
 61
 62#define RB_EVENT_HDR_SIZE 4
 63
 64int alloc_cpu_buffers(void)
 65{
 66	int i;
 67
 68	unsigned long buffer_size = oprofile_cpu_buffer_size;
 69	unsigned long byte_size = buffer_size * (sizeof(struct op_sample) +
 70						 RB_EVENT_HDR_SIZE);
 71
 72	op_ring_buffer = ring_buffer_alloc(byte_size, OP_BUFFER_FLAGS);
 73	if (!op_ring_buffer)
 74		goto fail;
 75
 76	for_each_possible_cpu(i) {
 77		struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
 78
 79		b->last_task = NULL;
 80		b->last_is_kernel = -1;
 81		b->tracing = 0;
 82		b->buffer_size = buffer_size;
 83		b->sample_received = 0;
 84		b->sample_lost_overflow = 0;
 85		b->backtrace_aborted = 0;
 86		b->sample_invalid_eip = 0;
 87		b->cpu = i;
 88		INIT_DELAYED_WORK(&b->work, wq_sync_buffer);
 89	}
 90	return 0;
 91
 92fail:
 93	free_cpu_buffers();
 94	return -ENOMEM;
 95}
 96
 97void start_cpu_work(void)
 98{
 99	int i;
100
101	work_enabled = 1;
102
103	for_each_online_cpu(i) {
104		struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
105
106		/*
107		 * Spread the work by 1 jiffy per cpu so they dont all
108		 * fire at once.
109		 */
110		schedule_delayed_work_on(i, &b->work, DEFAULT_TIMER_EXPIRE + i);
111	}
112}
113
114void end_cpu_work(void)
115{
116	work_enabled = 0;
117}
118
119void flush_cpu_work(void)
120{
121	int i;
122
123	for_each_online_cpu(i) {
124		struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
125
126		/* these works are per-cpu, no need for flush_sync */
127		flush_delayed_work(&b->work);
128	}
129}
130
131/*
132 * This function prepares the cpu buffer to write a sample.
133 *
134 * Struct op_entry is used during operations on the ring buffer while
135 * struct op_sample contains the data that is stored in the ring
136 * buffer. Struct entry can be uninitialized. The function reserves a
137 * data array that is specified by size. Use
138 * op_cpu_buffer_write_commit() after preparing the sample. In case of
139 * errors a null pointer is returned, otherwise the pointer to the
140 * sample.
141 *
142 */
143struct op_sample
144*op_cpu_buffer_write_reserve(struct op_entry *entry, unsigned long size)
145{
146	entry->event = ring_buffer_lock_reserve
147		(op_ring_buffer, sizeof(struct op_sample) +
148		 size * sizeof(entry->sample->data[0]));
149	if (!entry->event)
150		return NULL;
151	entry->sample = ring_buffer_event_data(entry->event);
152	entry->size = size;
153	entry->data = entry->sample->data;
154
155	return entry->sample;
156}
157
158int op_cpu_buffer_write_commit(struct op_entry *entry)
159{
160	return ring_buffer_unlock_commit(op_ring_buffer, entry->event);
161}
162
163struct op_sample *op_cpu_buffer_read_entry(struct op_entry *entry, int cpu)
164{
165	struct ring_buffer_event *e;
166	e = ring_buffer_consume(op_ring_buffer, cpu, NULL, NULL);
167	if (!e)
168		return NULL;
169
170	entry->event = e;
171	entry->sample = ring_buffer_event_data(e);
172	entry->size = (ring_buffer_event_length(e) - sizeof(struct op_sample))
173		/ sizeof(entry->sample->data[0]);
174	entry->data = entry->sample->data;
175	return entry->sample;
176}
177
178unsigned long op_cpu_buffer_entries(int cpu)
179{
180	return ring_buffer_entries_cpu(op_ring_buffer, cpu);
181}
182
183static int
184op_add_code(struct oprofile_cpu_buffer *cpu_buf, unsigned long backtrace,
185	    int is_kernel, struct task_struct *task)
186{
187	struct op_entry entry;
188	struct op_sample *sample;
189	unsigned long flags;
190	int size;
191
192	flags = 0;
193
194	if (backtrace)
195		flags |= TRACE_BEGIN;
196
197	/* notice a switch from user->kernel or vice versa */
198	is_kernel = !!is_kernel;
199	if (cpu_buf->last_is_kernel != is_kernel) {
200		cpu_buf->last_is_kernel = is_kernel;
201		flags |= KERNEL_CTX_SWITCH;
202		if (is_kernel)
203			flags |= IS_KERNEL;
204	}
205
206	/* notice a task switch */
207	if (cpu_buf->last_task != task) {
208		cpu_buf->last_task = task;
209		flags |= USER_CTX_SWITCH;
210	}
211
212	if (!flags)
213		/* nothing to do */
214		return 0;
215
216	if (flags & USER_CTX_SWITCH)
217		size = 1;
218	else
219		size = 0;
220
221	sample = op_cpu_buffer_write_reserve(&entry, size);
222	if (!sample)
223		return -ENOMEM;
224
225	sample->eip = ESCAPE_CODE;
226	sample->event = flags;
227
228	if (size)
229		op_cpu_buffer_add_data(&entry, (unsigned long)task);
230
231	op_cpu_buffer_write_commit(&entry);
232
233	return 0;
234}
235
236static inline int
237op_add_sample(struct oprofile_cpu_buffer *cpu_buf,
238	      unsigned long pc, unsigned long event)
239{
240	struct op_entry entry;
241	struct op_sample *sample;
242
243	sample = op_cpu_buffer_write_reserve(&entry, 0);
244	if (!sample)
245		return -ENOMEM;
246
247	sample->eip = pc;
248	sample->event = event;
249
250	return op_cpu_buffer_write_commit(&entry);
251}
252
253/*
254 * This must be safe from any context.
255 *
256 * is_kernel is needed because on some architectures you cannot
257 * tell if you are in kernel or user space simply by looking at
258 * pc. We tag this in the buffer by generating kernel enter/exit
259 * events whenever is_kernel changes
260 */
261static int
262log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc,
263	   unsigned long backtrace, int is_kernel, unsigned long event,
264	   struct task_struct *task)
265{
266	struct task_struct *tsk = task ? task : current;
267	cpu_buf->sample_received++;
268
269	if (pc == ESCAPE_CODE) {
270		cpu_buf->sample_invalid_eip++;
271		return 0;
272	}
273
274	if (op_add_code(cpu_buf, backtrace, is_kernel, tsk))
275		goto fail;
276
277	if (op_add_sample(cpu_buf, pc, event))
278		goto fail;
279
280	return 1;
281
282fail:
283	cpu_buf->sample_lost_overflow++;
284	return 0;
285}
286
287static inline void oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf)
288{
289	cpu_buf->tracing = 1;
290}
291
292static inline void oprofile_end_trace(struct oprofile_cpu_buffer *cpu_buf)
293{
294	cpu_buf->tracing = 0;
295}
296
297static inline void
298__oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
299			  unsigned long event, int is_kernel,
300			  struct task_struct *task)
301{
302	struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer);
303	unsigned long backtrace = oprofile_backtrace_depth;
304
305	/*
306	 * if log_sample() fail we can't backtrace since we lost the
307	 * source of this event
308	 */
309	if (!log_sample(cpu_buf, pc, backtrace, is_kernel, event, task))
310		/* failed */
311		return;
312
313	if (!backtrace)
314		return;
315
316	oprofile_begin_trace(cpu_buf);
317	oprofile_ops.backtrace(regs, backtrace);
318	oprofile_end_trace(cpu_buf);
319}
320
321void oprofile_add_ext_hw_sample(unsigned long pc, struct pt_regs * const regs,
322				unsigned long event, int is_kernel,
323				struct task_struct *task)
324{
325	__oprofile_add_ext_sample(pc, regs, event, is_kernel, task);
326}
327
328void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
329			     unsigned long event, int is_kernel)
330{
331	__oprofile_add_ext_sample(pc, regs, event, is_kernel, NULL);
332}
333
334void oprofile_add_sample(struct pt_regs * const regs, unsigned long event)
335{
336	int is_kernel;
337	unsigned long pc;
338
339	if (likely(regs)) {
340		is_kernel = !user_mode(regs);
341		pc = profile_pc(regs);
342	} else {
343		is_kernel = 0;    /* This value will not be used */
344		pc = ESCAPE_CODE; /* as this causes an early return. */
345	}
346
347	__oprofile_add_ext_sample(pc, regs, event, is_kernel, NULL);
348}
349
350/*
351 * Add samples with data to the ring buffer.
352 *
353 * Use oprofile_add_data(&entry, val) to add data and
354 * oprofile_write_commit(&entry) to commit the sample.
355 */
356void
357oprofile_write_reserve(struct op_entry *entry, struct pt_regs * const regs,
358		       unsigned long pc, int code, int size)
359{
360	struct op_sample *sample;
361	int is_kernel = !user_mode(regs);
362	struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer);
363
364	cpu_buf->sample_received++;
365
366	/* no backtraces for samples with data */
367	if (op_add_code(cpu_buf, 0, is_kernel, current))
368		goto fail;
369
370	sample = op_cpu_buffer_write_reserve(entry, size + 2);
371	if (!sample)
372		goto fail;
373	sample->eip = ESCAPE_CODE;
374	sample->event = 0;		/* no flags */
375
376	op_cpu_buffer_add_data(entry, code);
377	op_cpu_buffer_add_data(entry, pc);
378
379	return;
380
381fail:
382	entry->event = NULL;
383	cpu_buf->sample_lost_overflow++;
384}
385
386int oprofile_add_data(struct op_entry *entry, unsigned long val)
387{
388	if (!entry->event)
389		return 0;
390	return op_cpu_buffer_add_data(entry, val);
391}
392
393int oprofile_add_data64(struct op_entry *entry, u64 val)
394{
395	if (!entry->event)
396		return 0;
397	if (op_cpu_buffer_get_size(entry) < 2)
398		/*
399		 * the function returns 0 to indicate a too small
400		 * buffer, even if there is some space left
401		 */
402		return 0;
403	if (!op_cpu_buffer_add_data(entry, (u32)val))
404		return 0;
405	return op_cpu_buffer_add_data(entry, (u32)(val >> 32));
406}
407
408int oprofile_write_commit(struct op_entry *entry)
409{
410	if (!entry->event)
411		return -EINVAL;
412	return op_cpu_buffer_write_commit(entry);
413}
414
415void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event)
416{
417	struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer);
418	log_sample(cpu_buf, pc, 0, is_kernel, event, NULL);
419}
420
421void oprofile_add_trace(unsigned long pc)
422{
423	struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer);
424
425	if (!cpu_buf->tracing)
426		return;
427
428	/*
429	 * broken frame can give an eip with the same value as an
430	 * escape code, abort the trace if we get it
431	 */
432	if (pc == ESCAPE_CODE)
433		goto fail;
434
435	if (op_add_sample(cpu_buf, pc, 0))
436		goto fail;
437
438	return;
439fail:
440	cpu_buf->tracing = 0;
441	cpu_buf->backtrace_aborted++;
442	return;
443}
444
445/*
446 * This serves to avoid cpu buffer overflow, and makes sure
447 * the task mortuary progresses
448 *
449 * By using schedule_delayed_work_on and then schedule_delayed_work
450 * we guarantee this will stay on the correct cpu
451 */
452static void wq_sync_buffer(struct work_struct *work)
453{
454	struct oprofile_cpu_buffer *b =
455		container_of(work, struct oprofile_cpu_buffer, work.work);
456	if (b->cpu != smp_processor_id() && !cpu_online(b->cpu)) {
457		cancel_delayed_work(&b->work);
458		return;
459	}
460	sync_buffer(b->cpu);
461
462	/* don't re-add the work if we're shutting down */
463	if (work_enabled)
464		schedule_delayed_work(&b->work, DEFAULT_TIMER_EXPIRE);
465}