Loading...
1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
14#include <linux/ring_buffer.h>
15#include <generated/utsrelease.h>
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
20#include <linux/notifier.h>
21#include <linux/irqflags.h>
22#include <linux/debugfs.h>
23#include <linux/pagemap.h>
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
27#include <linux/kprobes.h>
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
31#include <linux/splice.h>
32#include <linux/kdebug.h>
33#include <linux/string.h>
34#include <linux/rwsem.h>
35#include <linux/slab.h>
36#include <linux/ctype.h>
37#include <linux/init.h>
38#include <linux/poll.h>
39#include <linux/nmi.h>
40#include <linux/fs.h>
41
42#include "trace.h"
43#include "trace_output.h"
44
45/*
46 * On boot up, the ring buffer is set to the minimum size, so that
47 * we do not waste memory on systems that are not using tracing.
48 */
49int ring_buffer_expanded;
50
51/*
52 * We need to change this state when a selftest is running.
53 * A selftest will lurk into the ring-buffer to count the
54 * entries inserted during the selftest although some concurrent
55 * insertions into the ring-buffer such as trace_printk could occurred
56 * at the same time, giving false positive or negative results.
57 */
58static bool __read_mostly tracing_selftest_running;
59
60/*
61 * If a tracer is running, we do not want to run SELFTEST.
62 */
63bool __read_mostly tracing_selftest_disabled;
64
65/* For tracers that don't implement custom flags */
66static struct tracer_opt dummy_tracer_opt[] = {
67 { }
68};
69
70static struct tracer_flags dummy_tracer_flags = {
71 .val = 0,
72 .opts = dummy_tracer_opt
73};
74
75static int dummy_set_flag(u32 old_flags, u32 bit, int set)
76{
77 return 0;
78}
79
80/*
81 * Kill all tracing for good (never come back).
82 * It is initialized to 1 but will turn to zero if the initialization
83 * of the tracer is successful. But that is the only place that sets
84 * this back to zero.
85 */
86static int tracing_disabled = 1;
87
88DEFINE_PER_CPU(int, ftrace_cpu_disabled);
89
90cpumask_var_t __read_mostly tracing_buffer_mask;
91
92/*
93 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
94 *
95 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
96 * is set, then ftrace_dump is called. This will output the contents
97 * of the ftrace buffers to the console. This is very useful for
98 * capturing traces that lead to crashes and outputing it to a
99 * serial console.
100 *
101 * It is default off, but you can enable it with either specifying
102 * "ftrace_dump_on_oops" in the kernel command line, or setting
103 * /proc/sys/kernel/ftrace_dump_on_oops
104 * Set 1 if you want to dump buffers of all CPUs
105 * Set 2 if you want to dump the buffer of the CPU that triggered oops
106 */
107
108enum ftrace_dump_mode ftrace_dump_on_oops;
109
110static int tracing_set_tracer(const char *buf);
111
112#define MAX_TRACER_SIZE 100
113static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
114static char *default_bootup_tracer;
115
116static int __init set_cmdline_ftrace(char *str)
117{
118 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
119 default_bootup_tracer = bootup_tracer_buf;
120 /* We are using ftrace early, expand it */
121 ring_buffer_expanded = 1;
122 return 1;
123}
124__setup("ftrace=", set_cmdline_ftrace);
125
126static int __init set_ftrace_dump_on_oops(char *str)
127{
128 if (*str++ != '=' || !*str) {
129 ftrace_dump_on_oops = DUMP_ALL;
130 return 1;
131 }
132
133 if (!strcmp("orig_cpu", str)) {
134 ftrace_dump_on_oops = DUMP_ORIG;
135 return 1;
136 }
137
138 return 0;
139}
140__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
141
142unsigned long long ns2usecs(cycle_t nsec)
143{
144 nsec += 500;
145 do_div(nsec, 1000);
146 return nsec;
147}
148
149/*
150 * The global_trace is the descriptor that holds the tracing
151 * buffers for the live tracing. For each CPU, it contains
152 * a link list of pages that will store trace entries. The
153 * page descriptor of the pages in the memory is used to hold
154 * the link list by linking the lru item in the page descriptor
155 * to each of the pages in the buffer per CPU.
156 *
157 * For each active CPU there is a data field that holds the
158 * pages for the buffer for that CPU. Each CPU has the same number
159 * of pages allocated for its buffer.
160 */
161static struct trace_array global_trace;
162
163static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
164
165int filter_current_check_discard(struct ring_buffer *buffer,
166 struct ftrace_event_call *call, void *rec,
167 struct ring_buffer_event *event)
168{
169 return filter_check_discard(call, rec, buffer, event);
170}
171EXPORT_SYMBOL_GPL(filter_current_check_discard);
172
173cycle_t ftrace_now(int cpu)
174{
175 u64 ts;
176
177 /* Early boot up does not have a buffer yet */
178 if (!global_trace.buffer)
179 return trace_clock_local();
180
181 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
182 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
183
184 return ts;
185}
186
187/*
188 * The max_tr is used to snapshot the global_trace when a maximum
189 * latency is reached. Some tracers will use this to store a maximum
190 * trace while it continues examining live traces.
191 *
192 * The buffers for the max_tr are set up the same as the global_trace.
193 * When a snapshot is taken, the link list of the max_tr is swapped
194 * with the link list of the global_trace and the buffers are reset for
195 * the global_trace so the tracing can continue.
196 */
197static struct trace_array max_tr;
198
199static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
200
201/* tracer_enabled is used to toggle activation of a tracer */
202static int tracer_enabled = 1;
203
204/**
205 * tracing_is_enabled - return tracer_enabled status
206 *
207 * This function is used by other tracers to know the status
208 * of the tracer_enabled flag. Tracers may use this function
209 * to know if it should enable their features when starting
210 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
211 */
212int tracing_is_enabled(void)
213{
214 return tracer_enabled;
215}
216
217/*
218 * trace_buf_size is the size in bytes that is allocated
219 * for a buffer. Note, the number of bytes is always rounded
220 * to page size.
221 *
222 * This number is purposely set to a low number of 16384.
223 * If the dump on oops happens, it will be much appreciated
224 * to not have to wait for all that output. Anyway this can be
225 * boot time and run time configurable.
226 */
227#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
228
229static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
230
231/* trace_types holds a link list of available tracers. */
232static struct tracer *trace_types __read_mostly;
233
234/* current_trace points to the tracer that is currently active */
235static struct tracer *current_trace __read_mostly;
236
237/*
238 * trace_types_lock is used to protect the trace_types list.
239 */
240static DEFINE_MUTEX(trace_types_lock);
241
242/*
243 * serialize the access of the ring buffer
244 *
245 * ring buffer serializes readers, but it is low level protection.
246 * The validity of the events (which returns by ring_buffer_peek() ..etc)
247 * are not protected by ring buffer.
248 *
249 * The content of events may become garbage if we allow other process consumes
250 * these events concurrently:
251 * A) the page of the consumed events may become a normal page
252 * (not reader page) in ring buffer, and this page will be rewrited
253 * by events producer.
254 * B) The page of the consumed events may become a page for splice_read,
255 * and this page will be returned to system.
256 *
257 * These primitives allow multi process access to different cpu ring buffer
258 * concurrently.
259 *
260 * These primitives don't distinguish read-only and read-consume access.
261 * Multi read-only access are also serialized.
262 */
263
264#ifdef CONFIG_SMP
265static DECLARE_RWSEM(all_cpu_access_lock);
266static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
267
268static inline void trace_access_lock(int cpu)
269{
270 if (cpu == TRACE_PIPE_ALL_CPU) {
271 /* gain it for accessing the whole ring buffer. */
272 down_write(&all_cpu_access_lock);
273 } else {
274 /* gain it for accessing a cpu ring buffer. */
275
276 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
277 down_read(&all_cpu_access_lock);
278
279 /* Secondly block other access to this @cpu ring buffer. */
280 mutex_lock(&per_cpu(cpu_access_lock, cpu));
281 }
282}
283
284static inline void trace_access_unlock(int cpu)
285{
286 if (cpu == TRACE_PIPE_ALL_CPU) {
287 up_write(&all_cpu_access_lock);
288 } else {
289 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
290 up_read(&all_cpu_access_lock);
291 }
292}
293
294static inline void trace_access_lock_init(void)
295{
296 int cpu;
297
298 for_each_possible_cpu(cpu)
299 mutex_init(&per_cpu(cpu_access_lock, cpu));
300}
301
302#else
303
304static DEFINE_MUTEX(access_lock);
305
306static inline void trace_access_lock(int cpu)
307{
308 (void)cpu;
309 mutex_lock(&access_lock);
310}
311
312static inline void trace_access_unlock(int cpu)
313{
314 (void)cpu;
315 mutex_unlock(&access_lock);
316}
317
318static inline void trace_access_lock_init(void)
319{
320}
321
322#endif
323
324/* trace_wait is a waitqueue for tasks blocked on trace_poll */
325static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
326
327/* trace_flags holds trace_options default values */
328unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
329 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
330 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
331 TRACE_ITER_IRQ_INFO;
332
333static int trace_stop_count;
334static DEFINE_RAW_SPINLOCK(tracing_start_lock);
335
336static void wakeup_work_handler(struct work_struct *work)
337{
338 wake_up(&trace_wait);
339}
340
341static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
342
343/**
344 * tracing_on - enable tracing buffers
345 *
346 * This function enables tracing buffers that may have been
347 * disabled with tracing_off.
348 */
349void tracing_on(void)
350{
351 if (global_trace.buffer)
352 ring_buffer_record_on(global_trace.buffer);
353 /*
354 * This flag is only looked at when buffers haven't been
355 * allocated yet. We don't really care about the race
356 * between setting this flag and actually turning
357 * on the buffer.
358 */
359 global_trace.buffer_disabled = 0;
360}
361EXPORT_SYMBOL_GPL(tracing_on);
362
363/**
364 * tracing_off - turn off tracing buffers
365 *
366 * This function stops the tracing buffers from recording data.
367 * It does not disable any overhead the tracers themselves may
368 * be causing. This function simply causes all recording to
369 * the ring buffers to fail.
370 */
371void tracing_off(void)
372{
373 if (global_trace.buffer)
374 ring_buffer_record_off(global_trace.buffer);
375 /*
376 * This flag is only looked at when buffers haven't been
377 * allocated yet. We don't really care about the race
378 * between setting this flag and actually turning
379 * on the buffer.
380 */
381 global_trace.buffer_disabled = 1;
382}
383EXPORT_SYMBOL_GPL(tracing_off);
384
385/**
386 * tracing_is_on - show state of ring buffers enabled
387 */
388int tracing_is_on(void)
389{
390 if (global_trace.buffer)
391 return ring_buffer_record_is_on(global_trace.buffer);
392 return !global_trace.buffer_disabled;
393}
394EXPORT_SYMBOL_GPL(tracing_is_on);
395
396/**
397 * trace_wake_up - wake up tasks waiting for trace input
398 *
399 * Schedules a delayed work to wake up any task that is blocked on the
400 * trace_wait queue. These is used with trace_poll for tasks polling the
401 * trace.
402 */
403void trace_wake_up(void)
404{
405 const unsigned long delay = msecs_to_jiffies(2);
406
407 if (trace_flags & TRACE_ITER_BLOCK)
408 return;
409 schedule_delayed_work(&wakeup_work, delay);
410}
411
412static int __init set_buf_size(char *str)
413{
414 unsigned long buf_size;
415
416 if (!str)
417 return 0;
418 buf_size = memparse(str, &str);
419 /* nr_entries can not be zero */
420 if (buf_size == 0)
421 return 0;
422 trace_buf_size = buf_size;
423 return 1;
424}
425__setup("trace_buf_size=", set_buf_size);
426
427static int __init set_tracing_thresh(char *str)
428{
429 unsigned long threshhold;
430 int ret;
431
432 if (!str)
433 return 0;
434 ret = strict_strtoul(str, 0, &threshhold);
435 if (ret < 0)
436 return 0;
437 tracing_thresh = threshhold * 1000;
438 return 1;
439}
440__setup("tracing_thresh=", set_tracing_thresh);
441
442unsigned long nsecs_to_usecs(unsigned long nsecs)
443{
444 return nsecs / 1000;
445}
446
447/* These must match the bit postions in trace_iterator_flags */
448static const char *trace_options[] = {
449 "print-parent",
450 "sym-offset",
451 "sym-addr",
452 "verbose",
453 "raw",
454 "hex",
455 "bin",
456 "block",
457 "stacktrace",
458 "trace_printk",
459 "ftrace_preempt",
460 "branch",
461 "annotate",
462 "userstacktrace",
463 "sym-userobj",
464 "printk-msg-only",
465 "context-info",
466 "latency-format",
467 "sleep-time",
468 "graph-time",
469 "record-cmd",
470 "overwrite",
471 "disable_on_free",
472 "irq-info",
473 NULL
474};
475
476static struct {
477 u64 (*func)(void);
478 const char *name;
479} trace_clocks[] = {
480 { trace_clock_local, "local" },
481 { trace_clock_global, "global" },
482 { trace_clock_counter, "counter" },
483};
484
485int trace_clock_id;
486
487/*
488 * trace_parser_get_init - gets the buffer for trace parser
489 */
490int trace_parser_get_init(struct trace_parser *parser, int size)
491{
492 memset(parser, 0, sizeof(*parser));
493
494 parser->buffer = kmalloc(size, GFP_KERNEL);
495 if (!parser->buffer)
496 return 1;
497
498 parser->size = size;
499 return 0;
500}
501
502/*
503 * trace_parser_put - frees the buffer for trace parser
504 */
505void trace_parser_put(struct trace_parser *parser)
506{
507 kfree(parser->buffer);
508}
509
510/*
511 * trace_get_user - reads the user input string separated by space
512 * (matched by isspace(ch))
513 *
514 * For each string found the 'struct trace_parser' is updated,
515 * and the function returns.
516 *
517 * Returns number of bytes read.
518 *
519 * See kernel/trace/trace.h for 'struct trace_parser' details.
520 */
521int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
522 size_t cnt, loff_t *ppos)
523{
524 char ch;
525 size_t read = 0;
526 ssize_t ret;
527
528 if (!*ppos)
529 trace_parser_clear(parser);
530
531 ret = get_user(ch, ubuf++);
532 if (ret)
533 goto out;
534
535 read++;
536 cnt--;
537
538 /*
539 * The parser is not finished with the last write,
540 * continue reading the user input without skipping spaces.
541 */
542 if (!parser->cont) {
543 /* skip white space */
544 while (cnt && isspace(ch)) {
545 ret = get_user(ch, ubuf++);
546 if (ret)
547 goto out;
548 read++;
549 cnt--;
550 }
551
552 /* only spaces were written */
553 if (isspace(ch)) {
554 *ppos += read;
555 ret = read;
556 goto out;
557 }
558
559 parser->idx = 0;
560 }
561
562 /* read the non-space input */
563 while (cnt && !isspace(ch)) {
564 if (parser->idx < parser->size - 1)
565 parser->buffer[parser->idx++] = ch;
566 else {
567 ret = -EINVAL;
568 goto out;
569 }
570 ret = get_user(ch, ubuf++);
571 if (ret)
572 goto out;
573 read++;
574 cnt--;
575 }
576
577 /* We either got finished input or we have to wait for another call. */
578 if (isspace(ch)) {
579 parser->buffer[parser->idx] = 0;
580 parser->cont = false;
581 } else {
582 parser->cont = true;
583 parser->buffer[parser->idx++] = ch;
584 }
585
586 *ppos += read;
587 ret = read;
588
589out:
590 return ret;
591}
592
593ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
594{
595 int len;
596 int ret;
597
598 if (!cnt)
599 return 0;
600
601 if (s->len <= s->readpos)
602 return -EBUSY;
603
604 len = s->len - s->readpos;
605 if (cnt > len)
606 cnt = len;
607 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
608 if (ret == cnt)
609 return -EFAULT;
610
611 cnt -= ret;
612
613 s->readpos += cnt;
614 return cnt;
615}
616
617static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
618{
619 int len;
620
621 if (s->len <= s->readpos)
622 return -EBUSY;
623
624 len = s->len - s->readpos;
625 if (cnt > len)
626 cnt = len;
627 memcpy(buf, s->buffer + s->readpos, cnt);
628
629 s->readpos += cnt;
630 return cnt;
631}
632
633/*
634 * ftrace_max_lock is used to protect the swapping of buffers
635 * when taking a max snapshot. The buffers themselves are
636 * protected by per_cpu spinlocks. But the action of the swap
637 * needs its own lock.
638 *
639 * This is defined as a arch_spinlock_t in order to help
640 * with performance when lockdep debugging is enabled.
641 *
642 * It is also used in other places outside the update_max_tr
643 * so it needs to be defined outside of the
644 * CONFIG_TRACER_MAX_TRACE.
645 */
646static arch_spinlock_t ftrace_max_lock =
647 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
648
649unsigned long __read_mostly tracing_thresh;
650
651#ifdef CONFIG_TRACER_MAX_TRACE
652unsigned long __read_mostly tracing_max_latency;
653
654/*
655 * Copy the new maximum trace into the separate maximum-trace
656 * structure. (this way the maximum trace is permanently saved,
657 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
658 */
659static void
660__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
661{
662 struct trace_array_cpu *data = tr->data[cpu];
663 struct trace_array_cpu *max_data;
664
665 max_tr.cpu = cpu;
666 max_tr.time_start = data->preempt_timestamp;
667
668 max_data = max_tr.data[cpu];
669 max_data->saved_latency = tracing_max_latency;
670 max_data->critical_start = data->critical_start;
671 max_data->critical_end = data->critical_end;
672
673 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
674 max_data->pid = tsk->pid;
675 max_data->uid = task_uid(tsk);
676 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
677 max_data->policy = tsk->policy;
678 max_data->rt_priority = tsk->rt_priority;
679
680 /* record this tasks comm */
681 tracing_record_cmdline(tsk);
682}
683
684/**
685 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
686 * @tr: tracer
687 * @tsk: the task with the latency
688 * @cpu: The cpu that initiated the trace.
689 *
690 * Flip the buffers between the @tr and the max_tr and record information
691 * about which task was the cause of this latency.
692 */
693void
694update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
695{
696 struct ring_buffer *buf = tr->buffer;
697
698 if (trace_stop_count)
699 return;
700
701 WARN_ON_ONCE(!irqs_disabled());
702 if (!current_trace->use_max_tr) {
703 WARN_ON_ONCE(1);
704 return;
705 }
706 arch_spin_lock(&ftrace_max_lock);
707
708 tr->buffer = max_tr.buffer;
709 max_tr.buffer = buf;
710
711 __update_max_tr(tr, tsk, cpu);
712 arch_spin_unlock(&ftrace_max_lock);
713}
714
715/**
716 * update_max_tr_single - only copy one trace over, and reset the rest
717 * @tr - tracer
718 * @tsk - task with the latency
719 * @cpu - the cpu of the buffer to copy.
720 *
721 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
722 */
723void
724update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
725{
726 int ret;
727
728 if (trace_stop_count)
729 return;
730
731 WARN_ON_ONCE(!irqs_disabled());
732 if (!current_trace->use_max_tr) {
733 WARN_ON_ONCE(1);
734 return;
735 }
736
737 arch_spin_lock(&ftrace_max_lock);
738
739 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
740
741 if (ret == -EBUSY) {
742 /*
743 * We failed to swap the buffer due to a commit taking
744 * place on this CPU. We fail to record, but we reset
745 * the max trace buffer (no one writes directly to it)
746 * and flag that it failed.
747 */
748 trace_array_printk(&max_tr, _THIS_IP_,
749 "Failed to swap buffers due to commit in progress\n");
750 }
751
752 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
753
754 __update_max_tr(tr, tsk, cpu);
755 arch_spin_unlock(&ftrace_max_lock);
756}
757#endif /* CONFIG_TRACER_MAX_TRACE */
758
759/**
760 * register_tracer - register a tracer with the ftrace system.
761 * @type - the plugin for the tracer
762 *
763 * Register a new plugin tracer.
764 */
765int register_tracer(struct tracer *type)
766{
767 struct tracer *t;
768 int ret = 0;
769
770 if (!type->name) {
771 pr_info("Tracer must have a name\n");
772 return -1;
773 }
774
775 if (strlen(type->name) >= MAX_TRACER_SIZE) {
776 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
777 return -1;
778 }
779
780 mutex_lock(&trace_types_lock);
781
782 tracing_selftest_running = true;
783
784 for (t = trace_types; t; t = t->next) {
785 if (strcmp(type->name, t->name) == 0) {
786 /* already found */
787 pr_info("Tracer %s already registered\n",
788 type->name);
789 ret = -1;
790 goto out;
791 }
792 }
793
794 if (!type->set_flag)
795 type->set_flag = &dummy_set_flag;
796 if (!type->flags)
797 type->flags = &dummy_tracer_flags;
798 else
799 if (!type->flags->opts)
800 type->flags->opts = dummy_tracer_opt;
801 if (!type->wait_pipe)
802 type->wait_pipe = default_wait_pipe;
803
804
805#ifdef CONFIG_FTRACE_STARTUP_TEST
806 if (type->selftest && !tracing_selftest_disabled) {
807 struct tracer *saved_tracer = current_trace;
808 struct trace_array *tr = &global_trace;
809
810 /*
811 * Run a selftest on this tracer.
812 * Here we reset the trace buffer, and set the current
813 * tracer to be this tracer. The tracer can then run some
814 * internal tracing to verify that everything is in order.
815 * If we fail, we do not register this tracer.
816 */
817 tracing_reset_online_cpus(tr);
818
819 current_trace = type;
820
821 /* If we expanded the buffers, make sure the max is expanded too */
822 if (ring_buffer_expanded && type->use_max_tr)
823 ring_buffer_resize(max_tr.buffer, trace_buf_size,
824 RING_BUFFER_ALL_CPUS);
825
826 /* the test is responsible for initializing and enabling */
827 pr_info("Testing tracer %s: ", type->name);
828 ret = type->selftest(type, tr);
829 /* the test is responsible for resetting too */
830 current_trace = saved_tracer;
831 if (ret) {
832 printk(KERN_CONT "FAILED!\n");
833 goto out;
834 }
835 /* Only reset on passing, to avoid touching corrupted buffers */
836 tracing_reset_online_cpus(tr);
837
838 /* Shrink the max buffer again */
839 if (ring_buffer_expanded && type->use_max_tr)
840 ring_buffer_resize(max_tr.buffer, 1,
841 RING_BUFFER_ALL_CPUS);
842
843 printk(KERN_CONT "PASSED\n");
844 }
845#endif
846
847 type->next = trace_types;
848 trace_types = type;
849
850 out:
851 tracing_selftest_running = false;
852 mutex_unlock(&trace_types_lock);
853
854 if (ret || !default_bootup_tracer)
855 goto out_unlock;
856
857 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
858 goto out_unlock;
859
860 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
861 /* Do we want this tracer to start on bootup? */
862 tracing_set_tracer(type->name);
863 default_bootup_tracer = NULL;
864 /* disable other selftests, since this will break it. */
865 tracing_selftest_disabled = 1;
866#ifdef CONFIG_FTRACE_STARTUP_TEST
867 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
868 type->name);
869#endif
870
871 out_unlock:
872 return ret;
873}
874
875void unregister_tracer(struct tracer *type)
876{
877 struct tracer **t;
878
879 mutex_lock(&trace_types_lock);
880 for (t = &trace_types; *t; t = &(*t)->next) {
881 if (*t == type)
882 goto found;
883 }
884 pr_info("Tracer %s not registered\n", type->name);
885 goto out;
886
887 found:
888 *t = (*t)->next;
889
890 if (type == current_trace && tracer_enabled) {
891 tracer_enabled = 0;
892 tracing_stop();
893 if (current_trace->stop)
894 current_trace->stop(&global_trace);
895 current_trace = &nop_trace;
896 }
897out:
898 mutex_unlock(&trace_types_lock);
899}
900
901void tracing_reset(struct trace_array *tr, int cpu)
902{
903 struct ring_buffer *buffer = tr->buffer;
904
905 ring_buffer_record_disable(buffer);
906
907 /* Make sure all commits have finished */
908 synchronize_sched();
909 ring_buffer_reset_cpu(buffer, cpu);
910
911 ring_buffer_record_enable(buffer);
912}
913
914void tracing_reset_online_cpus(struct trace_array *tr)
915{
916 struct ring_buffer *buffer = tr->buffer;
917 int cpu;
918
919 ring_buffer_record_disable(buffer);
920
921 /* Make sure all commits have finished */
922 synchronize_sched();
923
924 tr->time_start = ftrace_now(tr->cpu);
925
926 for_each_online_cpu(cpu)
927 ring_buffer_reset_cpu(buffer, cpu);
928
929 ring_buffer_record_enable(buffer);
930}
931
932void tracing_reset_current(int cpu)
933{
934 tracing_reset(&global_trace, cpu);
935}
936
937void tracing_reset_current_online_cpus(void)
938{
939 tracing_reset_online_cpus(&global_trace);
940}
941
942#define SAVED_CMDLINES 128
943#define NO_CMDLINE_MAP UINT_MAX
944static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
945static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
946static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
947static int cmdline_idx;
948static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
949
950/* temporary disable recording */
951static atomic_t trace_record_cmdline_disabled __read_mostly;
952
953static void trace_init_cmdlines(void)
954{
955 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
956 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
957 cmdline_idx = 0;
958}
959
960int is_tracing_stopped(void)
961{
962 return trace_stop_count;
963}
964
965/**
966 * ftrace_off_permanent - disable all ftrace code permanently
967 *
968 * This should only be called when a serious anomally has
969 * been detected. This will turn off the function tracing,
970 * ring buffers, and other tracing utilites. It takes no
971 * locks and can be called from any context.
972 */
973void ftrace_off_permanent(void)
974{
975 tracing_disabled = 1;
976 ftrace_stop();
977 tracing_off_permanent();
978}
979
980/**
981 * tracing_start - quick start of the tracer
982 *
983 * If tracing is enabled but was stopped by tracing_stop,
984 * this will start the tracer back up.
985 */
986void tracing_start(void)
987{
988 struct ring_buffer *buffer;
989 unsigned long flags;
990
991 if (tracing_disabled)
992 return;
993
994 raw_spin_lock_irqsave(&tracing_start_lock, flags);
995 if (--trace_stop_count) {
996 if (trace_stop_count < 0) {
997 /* Someone screwed up their debugging */
998 WARN_ON_ONCE(1);
999 trace_stop_count = 0;
1000 }
1001 goto out;
1002 }
1003
1004 /* Prevent the buffers from switching */
1005 arch_spin_lock(&ftrace_max_lock);
1006
1007 buffer = global_trace.buffer;
1008 if (buffer)
1009 ring_buffer_record_enable(buffer);
1010
1011 buffer = max_tr.buffer;
1012 if (buffer)
1013 ring_buffer_record_enable(buffer);
1014
1015 arch_spin_unlock(&ftrace_max_lock);
1016
1017 ftrace_start();
1018 out:
1019 raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1020}
1021
1022/**
1023 * tracing_stop - quick stop of the tracer
1024 *
1025 * Light weight way to stop tracing. Use in conjunction with
1026 * tracing_start.
1027 */
1028void tracing_stop(void)
1029{
1030 struct ring_buffer *buffer;
1031 unsigned long flags;
1032
1033 ftrace_stop();
1034 raw_spin_lock_irqsave(&tracing_start_lock, flags);
1035 if (trace_stop_count++)
1036 goto out;
1037
1038 /* Prevent the buffers from switching */
1039 arch_spin_lock(&ftrace_max_lock);
1040
1041 buffer = global_trace.buffer;
1042 if (buffer)
1043 ring_buffer_record_disable(buffer);
1044
1045 buffer = max_tr.buffer;
1046 if (buffer)
1047 ring_buffer_record_disable(buffer);
1048
1049 arch_spin_unlock(&ftrace_max_lock);
1050
1051 out:
1052 raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1053}
1054
1055void trace_stop_cmdline_recording(void);
1056
1057static void trace_save_cmdline(struct task_struct *tsk)
1058{
1059 unsigned pid, idx;
1060
1061 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1062 return;
1063
1064 /*
1065 * It's not the end of the world if we don't get
1066 * the lock, but we also don't want to spin
1067 * nor do we want to disable interrupts,
1068 * so if we miss here, then better luck next time.
1069 */
1070 if (!arch_spin_trylock(&trace_cmdline_lock))
1071 return;
1072
1073 idx = map_pid_to_cmdline[tsk->pid];
1074 if (idx == NO_CMDLINE_MAP) {
1075 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1076
1077 /*
1078 * Check whether the cmdline buffer at idx has a pid
1079 * mapped. We are going to overwrite that entry so we
1080 * need to clear the map_pid_to_cmdline. Otherwise we
1081 * would read the new comm for the old pid.
1082 */
1083 pid = map_cmdline_to_pid[idx];
1084 if (pid != NO_CMDLINE_MAP)
1085 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1086
1087 map_cmdline_to_pid[idx] = tsk->pid;
1088 map_pid_to_cmdline[tsk->pid] = idx;
1089
1090 cmdline_idx = idx;
1091 }
1092
1093 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1094
1095 arch_spin_unlock(&trace_cmdline_lock);
1096}
1097
1098void trace_find_cmdline(int pid, char comm[])
1099{
1100 unsigned map;
1101
1102 if (!pid) {
1103 strcpy(comm, "<idle>");
1104 return;
1105 }
1106
1107 if (WARN_ON_ONCE(pid < 0)) {
1108 strcpy(comm, "<XXX>");
1109 return;
1110 }
1111
1112 if (pid > PID_MAX_DEFAULT) {
1113 strcpy(comm, "<...>");
1114 return;
1115 }
1116
1117 preempt_disable();
1118 arch_spin_lock(&trace_cmdline_lock);
1119 map = map_pid_to_cmdline[pid];
1120 if (map != NO_CMDLINE_MAP)
1121 strcpy(comm, saved_cmdlines[map]);
1122 else
1123 strcpy(comm, "<...>");
1124
1125 arch_spin_unlock(&trace_cmdline_lock);
1126 preempt_enable();
1127}
1128
1129void tracing_record_cmdline(struct task_struct *tsk)
1130{
1131 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1132 !tracing_is_on())
1133 return;
1134
1135 trace_save_cmdline(tsk);
1136}
1137
1138void
1139tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1140 int pc)
1141{
1142 struct task_struct *tsk = current;
1143
1144 entry->preempt_count = pc & 0xff;
1145 entry->pid = (tsk) ? tsk->pid : 0;
1146 entry->padding = 0;
1147 entry->flags =
1148#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1149 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1150#else
1151 TRACE_FLAG_IRQS_NOSUPPORT |
1152#endif
1153 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1154 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1155 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1156}
1157EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1158
1159struct ring_buffer_event *
1160trace_buffer_lock_reserve(struct ring_buffer *buffer,
1161 int type,
1162 unsigned long len,
1163 unsigned long flags, int pc)
1164{
1165 struct ring_buffer_event *event;
1166
1167 event = ring_buffer_lock_reserve(buffer, len);
1168 if (event != NULL) {
1169 struct trace_entry *ent = ring_buffer_event_data(event);
1170
1171 tracing_generic_entry_update(ent, flags, pc);
1172 ent->type = type;
1173 }
1174
1175 return event;
1176}
1177
1178static inline void
1179__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1180 struct ring_buffer_event *event,
1181 unsigned long flags, int pc,
1182 int wake)
1183{
1184 ring_buffer_unlock_commit(buffer, event);
1185
1186 ftrace_trace_stack(buffer, flags, 6, pc);
1187 ftrace_trace_userstack(buffer, flags, pc);
1188
1189 if (wake)
1190 trace_wake_up();
1191}
1192
1193void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1194 struct ring_buffer_event *event,
1195 unsigned long flags, int pc)
1196{
1197 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1198}
1199
1200struct ring_buffer_event *
1201trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1202 int type, unsigned long len,
1203 unsigned long flags, int pc)
1204{
1205 *current_rb = global_trace.buffer;
1206 return trace_buffer_lock_reserve(*current_rb,
1207 type, len, flags, pc);
1208}
1209EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1210
1211void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1212 struct ring_buffer_event *event,
1213 unsigned long flags, int pc)
1214{
1215 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1216}
1217EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1218
1219void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1220 struct ring_buffer_event *event,
1221 unsigned long flags, int pc)
1222{
1223 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1224}
1225EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1226
1227void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1228 struct ring_buffer_event *event,
1229 unsigned long flags, int pc,
1230 struct pt_regs *regs)
1231{
1232 ring_buffer_unlock_commit(buffer, event);
1233
1234 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1235 ftrace_trace_userstack(buffer, flags, pc);
1236}
1237EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1238
1239void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1240 struct ring_buffer_event *event)
1241{
1242 ring_buffer_discard_commit(buffer, event);
1243}
1244EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1245
1246void
1247trace_function(struct trace_array *tr,
1248 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1249 int pc)
1250{
1251 struct ftrace_event_call *call = &event_function;
1252 struct ring_buffer *buffer = tr->buffer;
1253 struct ring_buffer_event *event;
1254 struct ftrace_entry *entry;
1255
1256 /* If we are reading the ring buffer, don't trace */
1257 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1258 return;
1259
1260 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1261 flags, pc);
1262 if (!event)
1263 return;
1264 entry = ring_buffer_event_data(event);
1265 entry->ip = ip;
1266 entry->parent_ip = parent_ip;
1267
1268 if (!filter_check_discard(call, entry, buffer, event))
1269 ring_buffer_unlock_commit(buffer, event);
1270}
1271
1272void
1273ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1274 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1275 int pc)
1276{
1277 if (likely(!atomic_read(&data->disabled)))
1278 trace_function(tr, ip, parent_ip, flags, pc);
1279}
1280
1281#ifdef CONFIG_STACKTRACE
1282
1283#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1284struct ftrace_stack {
1285 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1286};
1287
1288static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1289static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1290
1291static void __ftrace_trace_stack(struct ring_buffer *buffer,
1292 unsigned long flags,
1293 int skip, int pc, struct pt_regs *regs)
1294{
1295 struct ftrace_event_call *call = &event_kernel_stack;
1296 struct ring_buffer_event *event;
1297 struct stack_entry *entry;
1298 struct stack_trace trace;
1299 int use_stack;
1300 int size = FTRACE_STACK_ENTRIES;
1301
1302 trace.nr_entries = 0;
1303 trace.skip = skip;
1304
1305 /*
1306 * Since events can happen in NMIs there's no safe way to
1307 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1308 * or NMI comes in, it will just have to use the default
1309 * FTRACE_STACK_SIZE.
1310 */
1311 preempt_disable_notrace();
1312
1313 use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1314 /*
1315 * We don't need any atomic variables, just a barrier.
1316 * If an interrupt comes in, we don't care, because it would
1317 * have exited and put the counter back to what we want.
1318 * We just need a barrier to keep gcc from moving things
1319 * around.
1320 */
1321 barrier();
1322 if (use_stack == 1) {
1323 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1324 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1325
1326 if (regs)
1327 save_stack_trace_regs(regs, &trace);
1328 else
1329 save_stack_trace(&trace);
1330
1331 if (trace.nr_entries > size)
1332 size = trace.nr_entries;
1333 } else
1334 /* From now on, use_stack is a boolean */
1335 use_stack = 0;
1336
1337 size *= sizeof(unsigned long);
1338
1339 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1340 sizeof(*entry) + size, flags, pc);
1341 if (!event)
1342 goto out;
1343 entry = ring_buffer_event_data(event);
1344
1345 memset(&entry->caller, 0, size);
1346
1347 if (use_stack)
1348 memcpy(&entry->caller, trace.entries,
1349 trace.nr_entries * sizeof(unsigned long));
1350 else {
1351 trace.max_entries = FTRACE_STACK_ENTRIES;
1352 trace.entries = entry->caller;
1353 if (regs)
1354 save_stack_trace_regs(regs, &trace);
1355 else
1356 save_stack_trace(&trace);
1357 }
1358
1359 entry->size = trace.nr_entries;
1360
1361 if (!filter_check_discard(call, entry, buffer, event))
1362 ring_buffer_unlock_commit(buffer, event);
1363
1364 out:
1365 /* Again, don't let gcc optimize things here */
1366 barrier();
1367 __get_cpu_var(ftrace_stack_reserve)--;
1368 preempt_enable_notrace();
1369
1370}
1371
1372void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1373 int skip, int pc, struct pt_regs *regs)
1374{
1375 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1376 return;
1377
1378 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1379}
1380
1381void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1382 int skip, int pc)
1383{
1384 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1385 return;
1386
1387 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1388}
1389
1390void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1391 int pc)
1392{
1393 __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1394}
1395
1396/**
1397 * trace_dump_stack - record a stack back trace in the trace buffer
1398 */
1399void trace_dump_stack(void)
1400{
1401 unsigned long flags;
1402
1403 if (tracing_disabled || tracing_selftest_running)
1404 return;
1405
1406 local_save_flags(flags);
1407
1408 /* skipping 3 traces, seems to get us at the caller of this function */
1409 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1410}
1411
1412static DEFINE_PER_CPU(int, user_stack_count);
1413
1414void
1415ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1416{
1417 struct ftrace_event_call *call = &event_user_stack;
1418 struct ring_buffer_event *event;
1419 struct userstack_entry *entry;
1420 struct stack_trace trace;
1421
1422 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1423 return;
1424
1425 /*
1426 * NMIs can not handle page faults, even with fix ups.
1427 * The save user stack can (and often does) fault.
1428 */
1429 if (unlikely(in_nmi()))
1430 return;
1431
1432 /*
1433 * prevent recursion, since the user stack tracing may
1434 * trigger other kernel events.
1435 */
1436 preempt_disable();
1437 if (__this_cpu_read(user_stack_count))
1438 goto out;
1439
1440 __this_cpu_inc(user_stack_count);
1441
1442 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1443 sizeof(*entry), flags, pc);
1444 if (!event)
1445 goto out_drop_count;
1446 entry = ring_buffer_event_data(event);
1447
1448 entry->tgid = current->tgid;
1449 memset(&entry->caller, 0, sizeof(entry->caller));
1450
1451 trace.nr_entries = 0;
1452 trace.max_entries = FTRACE_STACK_ENTRIES;
1453 trace.skip = 0;
1454 trace.entries = entry->caller;
1455
1456 save_stack_trace_user(&trace);
1457 if (!filter_check_discard(call, entry, buffer, event))
1458 ring_buffer_unlock_commit(buffer, event);
1459
1460 out_drop_count:
1461 __this_cpu_dec(user_stack_count);
1462 out:
1463 preempt_enable();
1464}
1465
1466#ifdef UNUSED
1467static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1468{
1469 ftrace_trace_userstack(tr, flags, preempt_count());
1470}
1471#endif /* UNUSED */
1472
1473#endif /* CONFIG_STACKTRACE */
1474
1475/* created for use with alloc_percpu */
1476struct trace_buffer_struct {
1477 char buffer[TRACE_BUF_SIZE];
1478};
1479
1480static struct trace_buffer_struct *trace_percpu_buffer;
1481static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1482static struct trace_buffer_struct *trace_percpu_irq_buffer;
1483static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1484
1485/*
1486 * The buffer used is dependent on the context. There is a per cpu
1487 * buffer for normal context, softirq contex, hard irq context and
1488 * for NMI context. Thise allows for lockless recording.
1489 *
1490 * Note, if the buffers failed to be allocated, then this returns NULL
1491 */
1492static char *get_trace_buf(void)
1493{
1494 struct trace_buffer_struct *percpu_buffer;
1495 struct trace_buffer_struct *buffer;
1496
1497 /*
1498 * If we have allocated per cpu buffers, then we do not
1499 * need to do any locking.
1500 */
1501 if (in_nmi())
1502 percpu_buffer = trace_percpu_nmi_buffer;
1503 else if (in_irq())
1504 percpu_buffer = trace_percpu_irq_buffer;
1505 else if (in_softirq())
1506 percpu_buffer = trace_percpu_sirq_buffer;
1507 else
1508 percpu_buffer = trace_percpu_buffer;
1509
1510 if (!percpu_buffer)
1511 return NULL;
1512
1513 buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1514
1515 return buffer->buffer;
1516}
1517
1518static int alloc_percpu_trace_buffer(void)
1519{
1520 struct trace_buffer_struct *buffers;
1521 struct trace_buffer_struct *sirq_buffers;
1522 struct trace_buffer_struct *irq_buffers;
1523 struct trace_buffer_struct *nmi_buffers;
1524
1525 buffers = alloc_percpu(struct trace_buffer_struct);
1526 if (!buffers)
1527 goto err_warn;
1528
1529 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1530 if (!sirq_buffers)
1531 goto err_sirq;
1532
1533 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1534 if (!irq_buffers)
1535 goto err_irq;
1536
1537 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1538 if (!nmi_buffers)
1539 goto err_nmi;
1540
1541 trace_percpu_buffer = buffers;
1542 trace_percpu_sirq_buffer = sirq_buffers;
1543 trace_percpu_irq_buffer = irq_buffers;
1544 trace_percpu_nmi_buffer = nmi_buffers;
1545
1546 return 0;
1547
1548 err_nmi:
1549 free_percpu(irq_buffers);
1550 err_irq:
1551 free_percpu(sirq_buffers);
1552 err_sirq:
1553 free_percpu(buffers);
1554 err_warn:
1555 WARN(1, "Could not allocate percpu trace_printk buffer");
1556 return -ENOMEM;
1557}
1558
1559void trace_printk_init_buffers(void)
1560{
1561 static int buffers_allocated;
1562
1563 if (buffers_allocated)
1564 return;
1565
1566 if (alloc_percpu_trace_buffer())
1567 return;
1568
1569 pr_info("ftrace: Allocated trace_printk buffers\n");
1570
1571 buffers_allocated = 1;
1572}
1573
1574/**
1575 * trace_vbprintk - write binary msg to tracing buffer
1576 *
1577 */
1578int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1579{
1580 struct ftrace_event_call *call = &event_bprint;
1581 struct ring_buffer_event *event;
1582 struct ring_buffer *buffer;
1583 struct trace_array *tr = &global_trace;
1584 struct bprint_entry *entry;
1585 unsigned long flags;
1586 char *tbuffer;
1587 int len = 0, size, pc;
1588
1589 if (unlikely(tracing_selftest_running || tracing_disabled))
1590 return 0;
1591
1592 /* Don't pollute graph traces with trace_vprintk internals */
1593 pause_graph_tracing();
1594
1595 pc = preempt_count();
1596 preempt_disable_notrace();
1597
1598 tbuffer = get_trace_buf();
1599 if (!tbuffer) {
1600 len = 0;
1601 goto out;
1602 }
1603
1604 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1605
1606 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1607 goto out;
1608
1609 local_save_flags(flags);
1610 size = sizeof(*entry) + sizeof(u32) * len;
1611 buffer = tr->buffer;
1612 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1613 flags, pc);
1614 if (!event)
1615 goto out;
1616 entry = ring_buffer_event_data(event);
1617 entry->ip = ip;
1618 entry->fmt = fmt;
1619
1620 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1621 if (!filter_check_discard(call, entry, buffer, event)) {
1622 ring_buffer_unlock_commit(buffer, event);
1623 ftrace_trace_stack(buffer, flags, 6, pc);
1624 }
1625
1626out:
1627 preempt_enable_notrace();
1628 unpause_graph_tracing();
1629
1630 return len;
1631}
1632EXPORT_SYMBOL_GPL(trace_vbprintk);
1633
1634int trace_array_printk(struct trace_array *tr,
1635 unsigned long ip, const char *fmt, ...)
1636{
1637 int ret;
1638 va_list ap;
1639
1640 if (!(trace_flags & TRACE_ITER_PRINTK))
1641 return 0;
1642
1643 va_start(ap, fmt);
1644 ret = trace_array_vprintk(tr, ip, fmt, ap);
1645 va_end(ap);
1646 return ret;
1647}
1648
1649int trace_array_vprintk(struct trace_array *tr,
1650 unsigned long ip, const char *fmt, va_list args)
1651{
1652 struct ftrace_event_call *call = &event_print;
1653 struct ring_buffer_event *event;
1654 struct ring_buffer *buffer;
1655 int len = 0, size, pc;
1656 struct print_entry *entry;
1657 unsigned long flags;
1658 char *tbuffer;
1659
1660 if (tracing_disabled || tracing_selftest_running)
1661 return 0;
1662
1663 /* Don't pollute graph traces with trace_vprintk internals */
1664 pause_graph_tracing();
1665
1666 pc = preempt_count();
1667 preempt_disable_notrace();
1668
1669
1670 tbuffer = get_trace_buf();
1671 if (!tbuffer) {
1672 len = 0;
1673 goto out;
1674 }
1675
1676 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1677 if (len > TRACE_BUF_SIZE)
1678 goto out;
1679
1680 local_save_flags(flags);
1681 size = sizeof(*entry) + len + 1;
1682 buffer = tr->buffer;
1683 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1684 flags, pc);
1685 if (!event)
1686 goto out;
1687 entry = ring_buffer_event_data(event);
1688 entry->ip = ip;
1689
1690 memcpy(&entry->buf, tbuffer, len);
1691 entry->buf[len] = '\0';
1692 if (!filter_check_discard(call, entry, buffer, event)) {
1693 ring_buffer_unlock_commit(buffer, event);
1694 ftrace_trace_stack(buffer, flags, 6, pc);
1695 }
1696 out:
1697 preempt_enable_notrace();
1698 unpause_graph_tracing();
1699
1700 return len;
1701}
1702
1703int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1704{
1705 return trace_array_vprintk(&global_trace, ip, fmt, args);
1706}
1707EXPORT_SYMBOL_GPL(trace_vprintk);
1708
1709static void trace_iterator_increment(struct trace_iterator *iter)
1710{
1711 iter->idx++;
1712 if (iter->buffer_iter[iter->cpu])
1713 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1714}
1715
1716static struct trace_entry *
1717peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1718 unsigned long *lost_events)
1719{
1720 struct ring_buffer_event *event;
1721 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1722
1723 if (buf_iter)
1724 event = ring_buffer_iter_peek(buf_iter, ts);
1725 else
1726 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1727 lost_events);
1728
1729 if (event) {
1730 iter->ent_size = ring_buffer_event_length(event);
1731 return ring_buffer_event_data(event);
1732 }
1733 iter->ent_size = 0;
1734 return NULL;
1735}
1736
1737static struct trace_entry *
1738__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1739 unsigned long *missing_events, u64 *ent_ts)
1740{
1741 struct ring_buffer *buffer = iter->tr->buffer;
1742 struct trace_entry *ent, *next = NULL;
1743 unsigned long lost_events = 0, next_lost = 0;
1744 int cpu_file = iter->cpu_file;
1745 u64 next_ts = 0, ts;
1746 int next_cpu = -1;
1747 int next_size = 0;
1748 int cpu;
1749
1750 /*
1751 * If we are in a per_cpu trace file, don't bother by iterating over
1752 * all cpu and peek directly.
1753 */
1754 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1755 if (ring_buffer_empty_cpu(buffer, cpu_file))
1756 return NULL;
1757 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1758 if (ent_cpu)
1759 *ent_cpu = cpu_file;
1760
1761 return ent;
1762 }
1763
1764 for_each_tracing_cpu(cpu) {
1765
1766 if (ring_buffer_empty_cpu(buffer, cpu))
1767 continue;
1768
1769 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1770
1771 /*
1772 * Pick the entry with the smallest timestamp:
1773 */
1774 if (ent && (!next || ts < next_ts)) {
1775 next = ent;
1776 next_cpu = cpu;
1777 next_ts = ts;
1778 next_lost = lost_events;
1779 next_size = iter->ent_size;
1780 }
1781 }
1782
1783 iter->ent_size = next_size;
1784
1785 if (ent_cpu)
1786 *ent_cpu = next_cpu;
1787
1788 if (ent_ts)
1789 *ent_ts = next_ts;
1790
1791 if (missing_events)
1792 *missing_events = next_lost;
1793
1794 return next;
1795}
1796
1797/* Find the next real entry, without updating the iterator itself */
1798struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1799 int *ent_cpu, u64 *ent_ts)
1800{
1801 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1802}
1803
1804/* Find the next real entry, and increment the iterator to the next entry */
1805void *trace_find_next_entry_inc(struct trace_iterator *iter)
1806{
1807 iter->ent = __find_next_entry(iter, &iter->cpu,
1808 &iter->lost_events, &iter->ts);
1809
1810 if (iter->ent)
1811 trace_iterator_increment(iter);
1812
1813 return iter->ent ? iter : NULL;
1814}
1815
1816static void trace_consume(struct trace_iterator *iter)
1817{
1818 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1819 &iter->lost_events);
1820}
1821
1822static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1823{
1824 struct trace_iterator *iter = m->private;
1825 int i = (int)*pos;
1826 void *ent;
1827
1828 WARN_ON_ONCE(iter->leftover);
1829
1830 (*pos)++;
1831
1832 /* can't go backwards */
1833 if (iter->idx > i)
1834 return NULL;
1835
1836 if (iter->idx < 0)
1837 ent = trace_find_next_entry_inc(iter);
1838 else
1839 ent = iter;
1840
1841 while (ent && iter->idx < i)
1842 ent = trace_find_next_entry_inc(iter);
1843
1844 iter->pos = *pos;
1845
1846 return ent;
1847}
1848
1849void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1850{
1851 struct trace_array *tr = iter->tr;
1852 struct ring_buffer_event *event;
1853 struct ring_buffer_iter *buf_iter;
1854 unsigned long entries = 0;
1855 u64 ts;
1856
1857 tr->data[cpu]->skipped_entries = 0;
1858
1859 if (!iter->buffer_iter[cpu])
1860 return;
1861
1862 buf_iter = iter->buffer_iter[cpu];
1863 ring_buffer_iter_reset(buf_iter);
1864
1865 /*
1866 * We could have the case with the max latency tracers
1867 * that a reset never took place on a cpu. This is evident
1868 * by the timestamp being before the start of the buffer.
1869 */
1870 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1871 if (ts >= iter->tr->time_start)
1872 break;
1873 entries++;
1874 ring_buffer_read(buf_iter, NULL);
1875 }
1876
1877 tr->data[cpu]->skipped_entries = entries;
1878}
1879
1880/*
1881 * The current tracer is copied to avoid a global locking
1882 * all around.
1883 */
1884static void *s_start(struct seq_file *m, loff_t *pos)
1885{
1886 struct trace_iterator *iter = m->private;
1887 static struct tracer *old_tracer;
1888 int cpu_file = iter->cpu_file;
1889 void *p = NULL;
1890 loff_t l = 0;
1891 int cpu;
1892
1893 /* copy the tracer to avoid using a global lock all around */
1894 mutex_lock(&trace_types_lock);
1895 if (unlikely(old_tracer != current_trace && current_trace)) {
1896 old_tracer = current_trace;
1897 *iter->trace = *current_trace;
1898 }
1899 mutex_unlock(&trace_types_lock);
1900
1901 atomic_inc(&trace_record_cmdline_disabled);
1902
1903 if (*pos != iter->pos) {
1904 iter->ent = NULL;
1905 iter->cpu = 0;
1906 iter->idx = -1;
1907
1908 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1909 for_each_tracing_cpu(cpu)
1910 tracing_iter_reset(iter, cpu);
1911 } else
1912 tracing_iter_reset(iter, cpu_file);
1913
1914 iter->leftover = 0;
1915 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1916 ;
1917
1918 } else {
1919 /*
1920 * If we overflowed the seq_file before, then we want
1921 * to just reuse the trace_seq buffer again.
1922 */
1923 if (iter->leftover)
1924 p = iter;
1925 else {
1926 l = *pos - 1;
1927 p = s_next(m, p, &l);
1928 }
1929 }
1930
1931 trace_event_read_lock();
1932 trace_access_lock(cpu_file);
1933 return p;
1934}
1935
1936static void s_stop(struct seq_file *m, void *p)
1937{
1938 struct trace_iterator *iter = m->private;
1939
1940 atomic_dec(&trace_record_cmdline_disabled);
1941 trace_access_unlock(iter->cpu_file);
1942 trace_event_read_unlock();
1943}
1944
1945static void
1946get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
1947{
1948 unsigned long count;
1949 int cpu;
1950
1951 *total = 0;
1952 *entries = 0;
1953
1954 for_each_tracing_cpu(cpu) {
1955 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1956 /*
1957 * If this buffer has skipped entries, then we hold all
1958 * entries for the trace and we need to ignore the
1959 * ones before the time stamp.
1960 */
1961 if (tr->data[cpu]->skipped_entries) {
1962 count -= tr->data[cpu]->skipped_entries;
1963 /* total is the same as the entries */
1964 *total += count;
1965 } else
1966 *total += count +
1967 ring_buffer_overrun_cpu(tr->buffer, cpu);
1968 *entries += count;
1969 }
1970}
1971
1972static void print_lat_help_header(struct seq_file *m)
1973{
1974 seq_puts(m, "# _------=> CPU# \n");
1975 seq_puts(m, "# / _-----=> irqs-off \n");
1976 seq_puts(m, "# | / _----=> need-resched \n");
1977 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1978 seq_puts(m, "# ||| / _--=> preempt-depth \n");
1979 seq_puts(m, "# |||| / delay \n");
1980 seq_puts(m, "# cmd pid ||||| time | caller \n");
1981 seq_puts(m, "# \\ / ||||| \\ | / \n");
1982}
1983
1984static void print_event_info(struct trace_array *tr, struct seq_file *m)
1985{
1986 unsigned long total;
1987 unsigned long entries;
1988
1989 get_total_entries(tr, &total, &entries);
1990 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
1991 entries, total, num_online_cpus());
1992 seq_puts(m, "#\n");
1993}
1994
1995static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
1996{
1997 print_event_info(tr, m);
1998 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1999 seq_puts(m, "# | | | | |\n");
2000}
2001
2002static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
2003{
2004 print_event_info(tr, m);
2005 seq_puts(m, "# _-----=> irqs-off\n");
2006 seq_puts(m, "# / _----=> need-resched\n");
2007 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2008 seq_puts(m, "# || / _--=> preempt-depth\n");
2009 seq_puts(m, "# ||| / delay\n");
2010 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2011 seq_puts(m, "# | | | |||| | |\n");
2012}
2013
2014void
2015print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2016{
2017 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2018 struct trace_array *tr = iter->tr;
2019 struct trace_array_cpu *data = tr->data[tr->cpu];
2020 struct tracer *type = current_trace;
2021 unsigned long entries;
2022 unsigned long total;
2023 const char *name = "preemption";
2024
2025 if (type)
2026 name = type->name;
2027
2028 get_total_entries(tr, &total, &entries);
2029
2030 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2031 name, UTS_RELEASE);
2032 seq_puts(m, "# -----------------------------------"
2033 "---------------------------------\n");
2034 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2035 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2036 nsecs_to_usecs(data->saved_latency),
2037 entries,
2038 total,
2039 tr->cpu,
2040#if defined(CONFIG_PREEMPT_NONE)
2041 "server",
2042#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2043 "desktop",
2044#elif defined(CONFIG_PREEMPT)
2045 "preempt",
2046#else
2047 "unknown",
2048#endif
2049 /* These are reserved for later use */
2050 0, 0, 0, 0);
2051#ifdef CONFIG_SMP
2052 seq_printf(m, " #P:%d)\n", num_online_cpus());
2053#else
2054 seq_puts(m, ")\n");
2055#endif
2056 seq_puts(m, "# -----------------\n");
2057 seq_printf(m, "# | task: %.16s-%d "
2058 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2059 data->comm, data->pid, data->uid, data->nice,
2060 data->policy, data->rt_priority);
2061 seq_puts(m, "# -----------------\n");
2062
2063 if (data->critical_start) {
2064 seq_puts(m, "# => started at: ");
2065 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2066 trace_print_seq(m, &iter->seq);
2067 seq_puts(m, "\n# => ended at: ");
2068 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2069 trace_print_seq(m, &iter->seq);
2070 seq_puts(m, "\n#\n");
2071 }
2072
2073 seq_puts(m, "#\n");
2074}
2075
2076static void test_cpu_buff_start(struct trace_iterator *iter)
2077{
2078 struct trace_seq *s = &iter->seq;
2079
2080 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2081 return;
2082
2083 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2084 return;
2085
2086 if (cpumask_test_cpu(iter->cpu, iter->started))
2087 return;
2088
2089 if (iter->tr->data[iter->cpu]->skipped_entries)
2090 return;
2091
2092 cpumask_set_cpu(iter->cpu, iter->started);
2093
2094 /* Don't print started cpu buffer for the first entry of the trace */
2095 if (iter->idx > 1)
2096 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2097 iter->cpu);
2098}
2099
2100static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2101{
2102 struct trace_seq *s = &iter->seq;
2103 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2104 struct trace_entry *entry;
2105 struct trace_event *event;
2106
2107 entry = iter->ent;
2108
2109 test_cpu_buff_start(iter);
2110
2111 event = ftrace_find_event(entry->type);
2112
2113 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2114 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2115 if (!trace_print_lat_context(iter))
2116 goto partial;
2117 } else {
2118 if (!trace_print_context(iter))
2119 goto partial;
2120 }
2121 }
2122
2123 if (event)
2124 return event->funcs->trace(iter, sym_flags, event);
2125
2126 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2127 goto partial;
2128
2129 return TRACE_TYPE_HANDLED;
2130partial:
2131 return TRACE_TYPE_PARTIAL_LINE;
2132}
2133
2134static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2135{
2136 struct trace_seq *s = &iter->seq;
2137 struct trace_entry *entry;
2138 struct trace_event *event;
2139
2140 entry = iter->ent;
2141
2142 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2143 if (!trace_seq_printf(s, "%d %d %llu ",
2144 entry->pid, iter->cpu, iter->ts))
2145 goto partial;
2146 }
2147
2148 event = ftrace_find_event(entry->type);
2149 if (event)
2150 return event->funcs->raw(iter, 0, event);
2151
2152 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2153 goto partial;
2154
2155 return TRACE_TYPE_HANDLED;
2156partial:
2157 return TRACE_TYPE_PARTIAL_LINE;
2158}
2159
2160static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2161{
2162 struct trace_seq *s = &iter->seq;
2163 unsigned char newline = '\n';
2164 struct trace_entry *entry;
2165 struct trace_event *event;
2166
2167 entry = iter->ent;
2168
2169 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2170 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2171 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2172 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2173 }
2174
2175 event = ftrace_find_event(entry->type);
2176 if (event) {
2177 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2178 if (ret != TRACE_TYPE_HANDLED)
2179 return ret;
2180 }
2181
2182 SEQ_PUT_FIELD_RET(s, newline);
2183
2184 return TRACE_TYPE_HANDLED;
2185}
2186
2187static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2188{
2189 struct trace_seq *s = &iter->seq;
2190 struct trace_entry *entry;
2191 struct trace_event *event;
2192
2193 entry = iter->ent;
2194
2195 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2196 SEQ_PUT_FIELD_RET(s, entry->pid);
2197 SEQ_PUT_FIELD_RET(s, iter->cpu);
2198 SEQ_PUT_FIELD_RET(s, iter->ts);
2199 }
2200
2201 event = ftrace_find_event(entry->type);
2202 return event ? event->funcs->binary(iter, 0, event) :
2203 TRACE_TYPE_HANDLED;
2204}
2205
2206int trace_empty(struct trace_iterator *iter)
2207{
2208 int cpu;
2209
2210 /* If we are looking at one CPU buffer, only check that one */
2211 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2212 cpu = iter->cpu_file;
2213 if (iter->buffer_iter[cpu]) {
2214 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2215 return 0;
2216 } else {
2217 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2218 return 0;
2219 }
2220 return 1;
2221 }
2222
2223 for_each_tracing_cpu(cpu) {
2224 if (iter->buffer_iter[cpu]) {
2225 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2226 return 0;
2227 } else {
2228 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2229 return 0;
2230 }
2231 }
2232
2233 return 1;
2234}
2235
2236/* Called with trace_event_read_lock() held. */
2237enum print_line_t print_trace_line(struct trace_iterator *iter)
2238{
2239 enum print_line_t ret;
2240
2241 if (iter->lost_events &&
2242 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2243 iter->cpu, iter->lost_events))
2244 return TRACE_TYPE_PARTIAL_LINE;
2245
2246 if (iter->trace && iter->trace->print_line) {
2247 ret = iter->trace->print_line(iter);
2248 if (ret != TRACE_TYPE_UNHANDLED)
2249 return ret;
2250 }
2251
2252 if (iter->ent->type == TRACE_BPRINT &&
2253 trace_flags & TRACE_ITER_PRINTK &&
2254 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2255 return trace_print_bprintk_msg_only(iter);
2256
2257 if (iter->ent->type == TRACE_PRINT &&
2258 trace_flags & TRACE_ITER_PRINTK &&
2259 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2260 return trace_print_printk_msg_only(iter);
2261
2262 if (trace_flags & TRACE_ITER_BIN)
2263 return print_bin_fmt(iter);
2264
2265 if (trace_flags & TRACE_ITER_HEX)
2266 return print_hex_fmt(iter);
2267
2268 if (trace_flags & TRACE_ITER_RAW)
2269 return print_raw_fmt(iter);
2270
2271 return print_trace_fmt(iter);
2272}
2273
2274void trace_latency_header(struct seq_file *m)
2275{
2276 struct trace_iterator *iter = m->private;
2277
2278 /* print nothing if the buffers are empty */
2279 if (trace_empty(iter))
2280 return;
2281
2282 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2283 print_trace_header(m, iter);
2284
2285 if (!(trace_flags & TRACE_ITER_VERBOSE))
2286 print_lat_help_header(m);
2287}
2288
2289void trace_default_header(struct seq_file *m)
2290{
2291 struct trace_iterator *iter = m->private;
2292
2293 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2294 return;
2295
2296 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2297 /* print nothing if the buffers are empty */
2298 if (trace_empty(iter))
2299 return;
2300 print_trace_header(m, iter);
2301 if (!(trace_flags & TRACE_ITER_VERBOSE))
2302 print_lat_help_header(m);
2303 } else {
2304 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2305 if (trace_flags & TRACE_ITER_IRQ_INFO)
2306 print_func_help_header_irq(iter->tr, m);
2307 else
2308 print_func_help_header(iter->tr, m);
2309 }
2310 }
2311}
2312
2313static void test_ftrace_alive(struct seq_file *m)
2314{
2315 if (!ftrace_is_dead())
2316 return;
2317 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2318 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2319}
2320
2321static int s_show(struct seq_file *m, void *v)
2322{
2323 struct trace_iterator *iter = v;
2324 int ret;
2325
2326 if (iter->ent == NULL) {
2327 if (iter->tr) {
2328 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2329 seq_puts(m, "#\n");
2330 test_ftrace_alive(m);
2331 }
2332 if (iter->trace && iter->trace->print_header)
2333 iter->trace->print_header(m);
2334 else
2335 trace_default_header(m);
2336
2337 } else if (iter->leftover) {
2338 /*
2339 * If we filled the seq_file buffer earlier, we
2340 * want to just show it now.
2341 */
2342 ret = trace_print_seq(m, &iter->seq);
2343
2344 /* ret should this time be zero, but you never know */
2345 iter->leftover = ret;
2346
2347 } else {
2348 print_trace_line(iter);
2349 ret = trace_print_seq(m, &iter->seq);
2350 /*
2351 * If we overflow the seq_file buffer, then it will
2352 * ask us for this data again at start up.
2353 * Use that instead.
2354 * ret is 0 if seq_file write succeeded.
2355 * -1 otherwise.
2356 */
2357 iter->leftover = ret;
2358 }
2359
2360 return 0;
2361}
2362
2363static const struct seq_operations tracer_seq_ops = {
2364 .start = s_start,
2365 .next = s_next,
2366 .stop = s_stop,
2367 .show = s_show,
2368};
2369
2370static struct trace_iterator *
2371__tracing_open(struct inode *inode, struct file *file)
2372{
2373 long cpu_file = (long) inode->i_private;
2374 struct trace_iterator *iter;
2375 int cpu;
2376
2377 if (tracing_disabled)
2378 return ERR_PTR(-ENODEV);
2379
2380 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2381 if (!iter)
2382 return ERR_PTR(-ENOMEM);
2383
2384 /*
2385 * We make a copy of the current tracer to avoid concurrent
2386 * changes on it while we are reading.
2387 */
2388 mutex_lock(&trace_types_lock);
2389 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2390 if (!iter->trace)
2391 goto fail;
2392
2393 if (current_trace)
2394 *iter->trace = *current_trace;
2395
2396 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2397 goto fail;
2398
2399 if (current_trace && current_trace->print_max)
2400 iter->tr = &max_tr;
2401 else
2402 iter->tr = &global_trace;
2403 iter->pos = -1;
2404 mutex_init(&iter->mutex);
2405 iter->cpu_file = cpu_file;
2406
2407 /* Notify the tracer early; before we stop tracing. */
2408 if (iter->trace && iter->trace->open)
2409 iter->trace->open(iter);
2410
2411 /* Annotate start of buffers if we had overruns */
2412 if (ring_buffer_overruns(iter->tr->buffer))
2413 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2414
2415 /* stop the trace while dumping */
2416 tracing_stop();
2417
2418 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2419 for_each_tracing_cpu(cpu) {
2420 iter->buffer_iter[cpu] =
2421 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2422 }
2423 ring_buffer_read_prepare_sync();
2424 for_each_tracing_cpu(cpu) {
2425 ring_buffer_read_start(iter->buffer_iter[cpu]);
2426 tracing_iter_reset(iter, cpu);
2427 }
2428 } else {
2429 cpu = iter->cpu_file;
2430 iter->buffer_iter[cpu] =
2431 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2432 ring_buffer_read_prepare_sync();
2433 ring_buffer_read_start(iter->buffer_iter[cpu]);
2434 tracing_iter_reset(iter, cpu);
2435 }
2436
2437 mutex_unlock(&trace_types_lock);
2438
2439 return iter;
2440
2441 fail:
2442 mutex_unlock(&trace_types_lock);
2443 kfree(iter->trace);
2444 seq_release_private(inode, file);
2445 return ERR_PTR(-ENOMEM);
2446}
2447
2448int tracing_open_generic(struct inode *inode, struct file *filp)
2449{
2450 if (tracing_disabled)
2451 return -ENODEV;
2452
2453 filp->private_data = inode->i_private;
2454 return 0;
2455}
2456
2457static int tracing_release(struct inode *inode, struct file *file)
2458{
2459 struct seq_file *m = file->private_data;
2460 struct trace_iterator *iter;
2461 int cpu;
2462
2463 if (!(file->f_mode & FMODE_READ))
2464 return 0;
2465
2466 iter = m->private;
2467
2468 mutex_lock(&trace_types_lock);
2469 for_each_tracing_cpu(cpu) {
2470 if (iter->buffer_iter[cpu])
2471 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2472 }
2473
2474 if (iter->trace && iter->trace->close)
2475 iter->trace->close(iter);
2476
2477 /* reenable tracing if it was previously enabled */
2478 tracing_start();
2479 mutex_unlock(&trace_types_lock);
2480
2481 mutex_destroy(&iter->mutex);
2482 free_cpumask_var(iter->started);
2483 kfree(iter->trace);
2484 seq_release_private(inode, file);
2485 return 0;
2486}
2487
2488static int tracing_open(struct inode *inode, struct file *file)
2489{
2490 struct trace_iterator *iter;
2491 int ret = 0;
2492
2493 /* If this file was open for write, then erase contents */
2494 if ((file->f_mode & FMODE_WRITE) &&
2495 (file->f_flags & O_TRUNC)) {
2496 long cpu = (long) inode->i_private;
2497
2498 if (cpu == TRACE_PIPE_ALL_CPU)
2499 tracing_reset_online_cpus(&global_trace);
2500 else
2501 tracing_reset(&global_trace, cpu);
2502 }
2503
2504 if (file->f_mode & FMODE_READ) {
2505 iter = __tracing_open(inode, file);
2506 if (IS_ERR(iter))
2507 ret = PTR_ERR(iter);
2508 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2509 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2510 }
2511 return ret;
2512}
2513
2514static void *
2515t_next(struct seq_file *m, void *v, loff_t *pos)
2516{
2517 struct tracer *t = v;
2518
2519 (*pos)++;
2520
2521 if (t)
2522 t = t->next;
2523
2524 return t;
2525}
2526
2527static void *t_start(struct seq_file *m, loff_t *pos)
2528{
2529 struct tracer *t;
2530 loff_t l = 0;
2531
2532 mutex_lock(&trace_types_lock);
2533 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2534 ;
2535
2536 return t;
2537}
2538
2539static void t_stop(struct seq_file *m, void *p)
2540{
2541 mutex_unlock(&trace_types_lock);
2542}
2543
2544static int t_show(struct seq_file *m, void *v)
2545{
2546 struct tracer *t = v;
2547
2548 if (!t)
2549 return 0;
2550
2551 seq_printf(m, "%s", t->name);
2552 if (t->next)
2553 seq_putc(m, ' ');
2554 else
2555 seq_putc(m, '\n');
2556
2557 return 0;
2558}
2559
2560static const struct seq_operations show_traces_seq_ops = {
2561 .start = t_start,
2562 .next = t_next,
2563 .stop = t_stop,
2564 .show = t_show,
2565};
2566
2567static int show_traces_open(struct inode *inode, struct file *file)
2568{
2569 if (tracing_disabled)
2570 return -ENODEV;
2571
2572 return seq_open(file, &show_traces_seq_ops);
2573}
2574
2575static ssize_t
2576tracing_write_stub(struct file *filp, const char __user *ubuf,
2577 size_t count, loff_t *ppos)
2578{
2579 return count;
2580}
2581
2582static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2583{
2584 if (file->f_mode & FMODE_READ)
2585 return seq_lseek(file, offset, origin);
2586 else
2587 return 0;
2588}
2589
2590static const struct file_operations tracing_fops = {
2591 .open = tracing_open,
2592 .read = seq_read,
2593 .write = tracing_write_stub,
2594 .llseek = tracing_seek,
2595 .release = tracing_release,
2596};
2597
2598static const struct file_operations show_traces_fops = {
2599 .open = show_traces_open,
2600 .read = seq_read,
2601 .release = seq_release,
2602 .llseek = seq_lseek,
2603};
2604
2605/*
2606 * Only trace on a CPU if the bitmask is set:
2607 */
2608static cpumask_var_t tracing_cpumask;
2609
2610/*
2611 * The tracer itself will not take this lock, but still we want
2612 * to provide a consistent cpumask to user-space:
2613 */
2614static DEFINE_MUTEX(tracing_cpumask_update_lock);
2615
2616/*
2617 * Temporary storage for the character representation of the
2618 * CPU bitmask (and one more byte for the newline):
2619 */
2620static char mask_str[NR_CPUS + 1];
2621
2622static ssize_t
2623tracing_cpumask_read(struct file *filp, char __user *ubuf,
2624 size_t count, loff_t *ppos)
2625{
2626 int len;
2627
2628 mutex_lock(&tracing_cpumask_update_lock);
2629
2630 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2631 if (count - len < 2) {
2632 count = -EINVAL;
2633 goto out_err;
2634 }
2635 len += sprintf(mask_str + len, "\n");
2636 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2637
2638out_err:
2639 mutex_unlock(&tracing_cpumask_update_lock);
2640
2641 return count;
2642}
2643
2644static ssize_t
2645tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2646 size_t count, loff_t *ppos)
2647{
2648 int err, cpu;
2649 cpumask_var_t tracing_cpumask_new;
2650
2651 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2652 return -ENOMEM;
2653
2654 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2655 if (err)
2656 goto err_unlock;
2657
2658 mutex_lock(&tracing_cpumask_update_lock);
2659
2660 local_irq_disable();
2661 arch_spin_lock(&ftrace_max_lock);
2662 for_each_tracing_cpu(cpu) {
2663 /*
2664 * Increase/decrease the disabled counter if we are
2665 * about to flip a bit in the cpumask:
2666 */
2667 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2668 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2669 atomic_inc(&global_trace.data[cpu]->disabled);
2670 ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2671 }
2672 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2673 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2674 atomic_dec(&global_trace.data[cpu]->disabled);
2675 ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2676 }
2677 }
2678 arch_spin_unlock(&ftrace_max_lock);
2679 local_irq_enable();
2680
2681 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2682
2683 mutex_unlock(&tracing_cpumask_update_lock);
2684 free_cpumask_var(tracing_cpumask_new);
2685
2686 return count;
2687
2688err_unlock:
2689 free_cpumask_var(tracing_cpumask_new);
2690
2691 return err;
2692}
2693
2694static const struct file_operations tracing_cpumask_fops = {
2695 .open = tracing_open_generic,
2696 .read = tracing_cpumask_read,
2697 .write = tracing_cpumask_write,
2698 .llseek = generic_file_llseek,
2699};
2700
2701static int tracing_trace_options_show(struct seq_file *m, void *v)
2702{
2703 struct tracer_opt *trace_opts;
2704 u32 tracer_flags;
2705 int i;
2706
2707 mutex_lock(&trace_types_lock);
2708 tracer_flags = current_trace->flags->val;
2709 trace_opts = current_trace->flags->opts;
2710
2711 for (i = 0; trace_options[i]; i++) {
2712 if (trace_flags & (1 << i))
2713 seq_printf(m, "%s\n", trace_options[i]);
2714 else
2715 seq_printf(m, "no%s\n", trace_options[i]);
2716 }
2717
2718 for (i = 0; trace_opts[i].name; i++) {
2719 if (tracer_flags & trace_opts[i].bit)
2720 seq_printf(m, "%s\n", trace_opts[i].name);
2721 else
2722 seq_printf(m, "no%s\n", trace_opts[i].name);
2723 }
2724 mutex_unlock(&trace_types_lock);
2725
2726 return 0;
2727}
2728
2729static int __set_tracer_option(struct tracer *trace,
2730 struct tracer_flags *tracer_flags,
2731 struct tracer_opt *opts, int neg)
2732{
2733 int ret;
2734
2735 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2736 if (ret)
2737 return ret;
2738
2739 if (neg)
2740 tracer_flags->val &= ~opts->bit;
2741 else
2742 tracer_flags->val |= opts->bit;
2743 return 0;
2744}
2745
2746/* Try to assign a tracer specific option */
2747static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2748{
2749 struct tracer_flags *tracer_flags = trace->flags;
2750 struct tracer_opt *opts = NULL;
2751 int i;
2752
2753 for (i = 0; tracer_flags->opts[i].name; i++) {
2754 opts = &tracer_flags->opts[i];
2755
2756 if (strcmp(cmp, opts->name) == 0)
2757 return __set_tracer_option(trace, trace->flags,
2758 opts, neg);
2759 }
2760
2761 return -EINVAL;
2762}
2763
2764static void set_tracer_flags(unsigned int mask, int enabled)
2765{
2766 /* do nothing if flag is already set */
2767 if (!!(trace_flags & mask) == !!enabled)
2768 return;
2769
2770 if (enabled)
2771 trace_flags |= mask;
2772 else
2773 trace_flags &= ~mask;
2774
2775 if (mask == TRACE_ITER_RECORD_CMD)
2776 trace_event_enable_cmd_record(enabled);
2777
2778 if (mask == TRACE_ITER_OVERWRITE)
2779 ring_buffer_change_overwrite(global_trace.buffer, enabled);
2780}
2781
2782static ssize_t
2783tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2784 size_t cnt, loff_t *ppos)
2785{
2786 char buf[64];
2787 char *cmp;
2788 int neg = 0;
2789 int ret;
2790 int i;
2791
2792 if (cnt >= sizeof(buf))
2793 return -EINVAL;
2794
2795 if (copy_from_user(&buf, ubuf, cnt))
2796 return -EFAULT;
2797
2798 buf[cnt] = 0;
2799 cmp = strstrip(buf);
2800
2801 if (strncmp(cmp, "no", 2) == 0) {
2802 neg = 1;
2803 cmp += 2;
2804 }
2805
2806 for (i = 0; trace_options[i]; i++) {
2807 if (strcmp(cmp, trace_options[i]) == 0) {
2808 set_tracer_flags(1 << i, !neg);
2809 break;
2810 }
2811 }
2812
2813 /* If no option could be set, test the specific tracer options */
2814 if (!trace_options[i]) {
2815 mutex_lock(&trace_types_lock);
2816 ret = set_tracer_option(current_trace, cmp, neg);
2817 mutex_unlock(&trace_types_lock);
2818 if (ret)
2819 return ret;
2820 }
2821
2822 *ppos += cnt;
2823
2824 return cnt;
2825}
2826
2827static int tracing_trace_options_open(struct inode *inode, struct file *file)
2828{
2829 if (tracing_disabled)
2830 return -ENODEV;
2831 return single_open(file, tracing_trace_options_show, NULL);
2832}
2833
2834static const struct file_operations tracing_iter_fops = {
2835 .open = tracing_trace_options_open,
2836 .read = seq_read,
2837 .llseek = seq_lseek,
2838 .release = single_release,
2839 .write = tracing_trace_options_write,
2840};
2841
2842static const char readme_msg[] =
2843 "tracing mini-HOWTO:\n\n"
2844 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2845 "# cat /sys/kernel/debug/tracing/available_tracers\n"
2846 "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2847 "# cat /sys/kernel/debug/tracing/current_tracer\n"
2848 "nop\n"
2849 "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2850 "# cat /sys/kernel/debug/tracing/current_tracer\n"
2851 "wakeup\n"
2852 "# cat /sys/kernel/debug/tracing/trace_options\n"
2853 "noprint-parent nosym-offset nosym-addr noverbose\n"
2854 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2855 "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2856 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2857 "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
2858;
2859
2860static ssize_t
2861tracing_readme_read(struct file *filp, char __user *ubuf,
2862 size_t cnt, loff_t *ppos)
2863{
2864 return simple_read_from_buffer(ubuf, cnt, ppos,
2865 readme_msg, strlen(readme_msg));
2866}
2867
2868static const struct file_operations tracing_readme_fops = {
2869 .open = tracing_open_generic,
2870 .read = tracing_readme_read,
2871 .llseek = generic_file_llseek,
2872};
2873
2874static ssize_t
2875tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2876 size_t cnt, loff_t *ppos)
2877{
2878 char *buf_comm;
2879 char *file_buf;
2880 char *buf;
2881 int len = 0;
2882 int pid;
2883 int i;
2884
2885 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2886 if (!file_buf)
2887 return -ENOMEM;
2888
2889 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2890 if (!buf_comm) {
2891 kfree(file_buf);
2892 return -ENOMEM;
2893 }
2894
2895 buf = file_buf;
2896
2897 for (i = 0; i < SAVED_CMDLINES; i++) {
2898 int r;
2899
2900 pid = map_cmdline_to_pid[i];
2901 if (pid == -1 || pid == NO_CMDLINE_MAP)
2902 continue;
2903
2904 trace_find_cmdline(pid, buf_comm);
2905 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2906 buf += r;
2907 len += r;
2908 }
2909
2910 len = simple_read_from_buffer(ubuf, cnt, ppos,
2911 file_buf, len);
2912
2913 kfree(file_buf);
2914 kfree(buf_comm);
2915
2916 return len;
2917}
2918
2919static const struct file_operations tracing_saved_cmdlines_fops = {
2920 .open = tracing_open_generic,
2921 .read = tracing_saved_cmdlines_read,
2922 .llseek = generic_file_llseek,
2923};
2924
2925static ssize_t
2926tracing_ctrl_read(struct file *filp, char __user *ubuf,
2927 size_t cnt, loff_t *ppos)
2928{
2929 char buf[64];
2930 int r;
2931
2932 r = sprintf(buf, "%u\n", tracer_enabled);
2933 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2934}
2935
2936static ssize_t
2937tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2938 size_t cnt, loff_t *ppos)
2939{
2940 struct trace_array *tr = filp->private_data;
2941 unsigned long val;
2942 int ret;
2943
2944 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2945 if (ret)
2946 return ret;
2947
2948 val = !!val;
2949
2950 mutex_lock(&trace_types_lock);
2951 if (tracer_enabled ^ val) {
2952
2953 /* Only need to warn if this is used to change the state */
2954 WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2955
2956 if (val) {
2957 tracer_enabled = 1;
2958 if (current_trace->start)
2959 current_trace->start(tr);
2960 tracing_start();
2961 } else {
2962 tracer_enabled = 0;
2963 tracing_stop();
2964 if (current_trace->stop)
2965 current_trace->stop(tr);
2966 }
2967 }
2968 mutex_unlock(&trace_types_lock);
2969
2970 *ppos += cnt;
2971
2972 return cnt;
2973}
2974
2975static ssize_t
2976tracing_set_trace_read(struct file *filp, char __user *ubuf,
2977 size_t cnt, loff_t *ppos)
2978{
2979 char buf[MAX_TRACER_SIZE+2];
2980 int r;
2981
2982 mutex_lock(&trace_types_lock);
2983 if (current_trace)
2984 r = sprintf(buf, "%s\n", current_trace->name);
2985 else
2986 r = sprintf(buf, "\n");
2987 mutex_unlock(&trace_types_lock);
2988
2989 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2990}
2991
2992int tracer_init(struct tracer *t, struct trace_array *tr)
2993{
2994 tracing_reset_online_cpus(tr);
2995 return t->init(tr);
2996}
2997
2998static void set_buffer_entries(struct trace_array *tr, unsigned long val)
2999{
3000 int cpu;
3001 for_each_tracing_cpu(cpu)
3002 tr->data[cpu]->entries = val;
3003}
3004
3005static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
3006{
3007 int ret;
3008
3009 /*
3010 * If kernel or user changes the size of the ring buffer
3011 * we use the size that was given, and we can forget about
3012 * expanding it later.
3013 */
3014 ring_buffer_expanded = 1;
3015
3016 ret = ring_buffer_resize(global_trace.buffer, size, cpu);
3017 if (ret < 0)
3018 return ret;
3019
3020 if (!current_trace->use_max_tr)
3021 goto out;
3022
3023 ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3024 if (ret < 0) {
3025 int r = 0;
3026
3027 if (cpu == RING_BUFFER_ALL_CPUS) {
3028 int i;
3029 for_each_tracing_cpu(i) {
3030 r = ring_buffer_resize(global_trace.buffer,
3031 global_trace.data[i]->entries,
3032 i);
3033 if (r < 0)
3034 break;
3035 }
3036 } else {
3037 r = ring_buffer_resize(global_trace.buffer,
3038 global_trace.data[cpu]->entries,
3039 cpu);
3040 }
3041
3042 if (r < 0) {
3043 /*
3044 * AARGH! We are left with different
3045 * size max buffer!!!!
3046 * The max buffer is our "snapshot" buffer.
3047 * When a tracer needs a snapshot (one of the
3048 * latency tracers), it swaps the max buffer
3049 * with the saved snap shot. We succeeded to
3050 * update the size of the main buffer, but failed to
3051 * update the size of the max buffer. But when we tried
3052 * to reset the main buffer to the original size, we
3053 * failed there too. This is very unlikely to
3054 * happen, but if it does, warn and kill all
3055 * tracing.
3056 */
3057 WARN_ON(1);
3058 tracing_disabled = 1;
3059 }
3060 return ret;
3061 }
3062
3063 if (cpu == RING_BUFFER_ALL_CPUS)
3064 set_buffer_entries(&max_tr, size);
3065 else
3066 max_tr.data[cpu]->entries = size;
3067
3068 out:
3069 if (cpu == RING_BUFFER_ALL_CPUS)
3070 set_buffer_entries(&global_trace, size);
3071 else
3072 global_trace.data[cpu]->entries = size;
3073
3074 return ret;
3075}
3076
3077static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3078{
3079 int ret = size;
3080
3081 mutex_lock(&trace_types_lock);
3082
3083 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3084 /* make sure, this cpu is enabled in the mask */
3085 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3086 ret = -EINVAL;
3087 goto out;
3088 }
3089 }
3090
3091 ret = __tracing_resize_ring_buffer(size, cpu_id);
3092 if (ret < 0)
3093 ret = -ENOMEM;
3094
3095out:
3096 mutex_unlock(&trace_types_lock);
3097
3098 return ret;
3099}
3100
3101
3102/**
3103 * tracing_update_buffers - used by tracing facility to expand ring buffers
3104 *
3105 * To save on memory when the tracing is never used on a system with it
3106 * configured in. The ring buffers are set to a minimum size. But once
3107 * a user starts to use the tracing facility, then they need to grow
3108 * to their default size.
3109 *
3110 * This function is to be called when a tracer is about to be used.
3111 */
3112int tracing_update_buffers(void)
3113{
3114 int ret = 0;
3115
3116 mutex_lock(&trace_types_lock);
3117 if (!ring_buffer_expanded)
3118 ret = __tracing_resize_ring_buffer(trace_buf_size,
3119 RING_BUFFER_ALL_CPUS);
3120 mutex_unlock(&trace_types_lock);
3121
3122 return ret;
3123}
3124
3125struct trace_option_dentry;
3126
3127static struct trace_option_dentry *
3128create_trace_option_files(struct tracer *tracer);
3129
3130static void
3131destroy_trace_option_files(struct trace_option_dentry *topts);
3132
3133static int tracing_set_tracer(const char *buf)
3134{
3135 static struct trace_option_dentry *topts;
3136 struct trace_array *tr = &global_trace;
3137 struct tracer *t;
3138 int ret = 0;
3139
3140 mutex_lock(&trace_types_lock);
3141
3142 if (!ring_buffer_expanded) {
3143 ret = __tracing_resize_ring_buffer(trace_buf_size,
3144 RING_BUFFER_ALL_CPUS);
3145 if (ret < 0)
3146 goto out;
3147 ret = 0;
3148 }
3149
3150 for (t = trace_types; t; t = t->next) {
3151 if (strcmp(t->name, buf) == 0)
3152 break;
3153 }
3154 if (!t) {
3155 ret = -EINVAL;
3156 goto out;
3157 }
3158 if (t == current_trace)
3159 goto out;
3160
3161 trace_branch_disable();
3162 if (current_trace && current_trace->reset)
3163 current_trace->reset(tr);
3164 if (current_trace && current_trace->use_max_tr) {
3165 /*
3166 * We don't free the ring buffer. instead, resize it because
3167 * The max_tr ring buffer has some state (e.g. ring->clock) and
3168 * we want preserve it.
3169 */
3170 ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3171 set_buffer_entries(&max_tr, 1);
3172 }
3173 destroy_trace_option_files(topts);
3174
3175 current_trace = t;
3176
3177 topts = create_trace_option_files(current_trace);
3178 if (current_trace->use_max_tr) {
3179 int cpu;
3180 /* we need to make per cpu buffer sizes equivalent */
3181 for_each_tracing_cpu(cpu) {
3182 ret = ring_buffer_resize(max_tr.buffer,
3183 global_trace.data[cpu]->entries,
3184 cpu);
3185 if (ret < 0)
3186 goto out;
3187 max_tr.data[cpu]->entries =
3188 global_trace.data[cpu]->entries;
3189 }
3190 }
3191
3192 if (t->init) {
3193 ret = tracer_init(t, tr);
3194 if (ret)
3195 goto out;
3196 }
3197
3198 trace_branch_enable(tr);
3199 out:
3200 mutex_unlock(&trace_types_lock);
3201
3202 return ret;
3203}
3204
3205static ssize_t
3206tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3207 size_t cnt, loff_t *ppos)
3208{
3209 char buf[MAX_TRACER_SIZE+1];
3210 int i;
3211 size_t ret;
3212 int err;
3213
3214 ret = cnt;
3215
3216 if (cnt > MAX_TRACER_SIZE)
3217 cnt = MAX_TRACER_SIZE;
3218
3219 if (copy_from_user(&buf, ubuf, cnt))
3220 return -EFAULT;
3221
3222 buf[cnt] = 0;
3223
3224 /* strip ending whitespace. */
3225 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3226 buf[i] = 0;
3227
3228 err = tracing_set_tracer(buf);
3229 if (err)
3230 return err;
3231
3232 *ppos += ret;
3233
3234 return ret;
3235}
3236
3237static ssize_t
3238tracing_max_lat_read(struct file *filp, char __user *ubuf,
3239 size_t cnt, loff_t *ppos)
3240{
3241 unsigned long *ptr = filp->private_data;
3242 char buf[64];
3243 int r;
3244
3245 r = snprintf(buf, sizeof(buf), "%ld\n",
3246 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3247 if (r > sizeof(buf))
3248 r = sizeof(buf);
3249 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3250}
3251
3252static ssize_t
3253tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3254 size_t cnt, loff_t *ppos)
3255{
3256 unsigned long *ptr = filp->private_data;
3257 unsigned long val;
3258 int ret;
3259
3260 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3261 if (ret)
3262 return ret;
3263
3264 *ptr = val * 1000;
3265
3266 return cnt;
3267}
3268
3269static int tracing_open_pipe(struct inode *inode, struct file *filp)
3270{
3271 long cpu_file = (long) inode->i_private;
3272 struct trace_iterator *iter;
3273 int ret = 0;
3274
3275 if (tracing_disabled)
3276 return -ENODEV;
3277
3278 mutex_lock(&trace_types_lock);
3279
3280 /* create a buffer to store the information to pass to userspace */
3281 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3282 if (!iter) {
3283 ret = -ENOMEM;
3284 goto out;
3285 }
3286
3287 /*
3288 * We make a copy of the current tracer to avoid concurrent
3289 * changes on it while we are reading.
3290 */
3291 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3292 if (!iter->trace) {
3293 ret = -ENOMEM;
3294 goto fail;
3295 }
3296 if (current_trace)
3297 *iter->trace = *current_trace;
3298
3299 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3300 ret = -ENOMEM;
3301 goto fail;
3302 }
3303
3304 /* trace pipe does not show start of buffer */
3305 cpumask_setall(iter->started);
3306
3307 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3308 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3309
3310 iter->cpu_file = cpu_file;
3311 iter->tr = &global_trace;
3312 mutex_init(&iter->mutex);
3313 filp->private_data = iter;
3314
3315 if (iter->trace->pipe_open)
3316 iter->trace->pipe_open(iter);
3317
3318 nonseekable_open(inode, filp);
3319out:
3320 mutex_unlock(&trace_types_lock);
3321 return ret;
3322
3323fail:
3324 kfree(iter->trace);
3325 kfree(iter);
3326 mutex_unlock(&trace_types_lock);
3327 return ret;
3328}
3329
3330static int tracing_release_pipe(struct inode *inode, struct file *file)
3331{
3332 struct trace_iterator *iter = file->private_data;
3333
3334 mutex_lock(&trace_types_lock);
3335
3336 if (iter->trace->pipe_close)
3337 iter->trace->pipe_close(iter);
3338
3339 mutex_unlock(&trace_types_lock);
3340
3341 free_cpumask_var(iter->started);
3342 mutex_destroy(&iter->mutex);
3343 kfree(iter->trace);
3344 kfree(iter);
3345
3346 return 0;
3347}
3348
3349static unsigned int
3350tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3351{
3352 struct trace_iterator *iter = filp->private_data;
3353
3354 if (trace_flags & TRACE_ITER_BLOCK) {
3355 /*
3356 * Always select as readable when in blocking mode
3357 */
3358 return POLLIN | POLLRDNORM;
3359 } else {
3360 if (!trace_empty(iter))
3361 return POLLIN | POLLRDNORM;
3362 poll_wait(filp, &trace_wait, poll_table);
3363 if (!trace_empty(iter))
3364 return POLLIN | POLLRDNORM;
3365
3366 return 0;
3367 }
3368}
3369
3370
3371void default_wait_pipe(struct trace_iterator *iter)
3372{
3373 DEFINE_WAIT(wait);
3374
3375 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3376
3377 if (trace_empty(iter))
3378 schedule();
3379
3380 finish_wait(&trace_wait, &wait);
3381}
3382
3383/*
3384 * This is a make-shift waitqueue.
3385 * A tracer might use this callback on some rare cases:
3386 *
3387 * 1) the current tracer might hold the runqueue lock when it wakes up
3388 * a reader, hence a deadlock (sched, function, and function graph tracers)
3389 * 2) the function tracers, trace all functions, we don't want
3390 * the overhead of calling wake_up and friends
3391 * (and tracing them too)
3392 *
3393 * Anyway, this is really very primitive wakeup.
3394 */
3395void poll_wait_pipe(struct trace_iterator *iter)
3396{
3397 set_current_state(TASK_INTERRUPTIBLE);
3398 /* sleep for 100 msecs, and try again. */
3399 schedule_timeout(HZ / 10);
3400}
3401
3402/* Must be called with trace_types_lock mutex held. */
3403static int tracing_wait_pipe(struct file *filp)
3404{
3405 struct trace_iterator *iter = filp->private_data;
3406
3407 while (trace_empty(iter)) {
3408
3409 if ((filp->f_flags & O_NONBLOCK)) {
3410 return -EAGAIN;
3411 }
3412
3413 mutex_unlock(&iter->mutex);
3414
3415 iter->trace->wait_pipe(iter);
3416
3417 mutex_lock(&iter->mutex);
3418
3419 if (signal_pending(current))
3420 return -EINTR;
3421
3422 /*
3423 * We block until we read something and tracing is disabled.
3424 * We still block if tracing is disabled, but we have never
3425 * read anything. This allows a user to cat this file, and
3426 * then enable tracing. But after we have read something,
3427 * we give an EOF when tracing is again disabled.
3428 *
3429 * iter->pos will be 0 if we haven't read anything.
3430 */
3431 if (!tracer_enabled && iter->pos)
3432 break;
3433 }
3434
3435 return 1;
3436}
3437
3438/*
3439 * Consumer reader.
3440 */
3441static ssize_t
3442tracing_read_pipe(struct file *filp, char __user *ubuf,
3443 size_t cnt, loff_t *ppos)
3444{
3445 struct trace_iterator *iter = filp->private_data;
3446 static struct tracer *old_tracer;
3447 ssize_t sret;
3448
3449 /* return any leftover data */
3450 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3451 if (sret != -EBUSY)
3452 return sret;
3453
3454 trace_seq_init(&iter->seq);
3455
3456 /* copy the tracer to avoid using a global lock all around */
3457 mutex_lock(&trace_types_lock);
3458 if (unlikely(old_tracer != current_trace && current_trace)) {
3459 old_tracer = current_trace;
3460 *iter->trace = *current_trace;
3461 }
3462 mutex_unlock(&trace_types_lock);
3463
3464 /*
3465 * Avoid more than one consumer on a single file descriptor
3466 * This is just a matter of traces coherency, the ring buffer itself
3467 * is protected.
3468 */
3469 mutex_lock(&iter->mutex);
3470 if (iter->trace->read) {
3471 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3472 if (sret)
3473 goto out;
3474 }
3475
3476waitagain:
3477 sret = tracing_wait_pipe(filp);
3478 if (sret <= 0)
3479 goto out;
3480
3481 /* stop when tracing is finished */
3482 if (trace_empty(iter)) {
3483 sret = 0;
3484 goto out;
3485 }
3486
3487 if (cnt >= PAGE_SIZE)
3488 cnt = PAGE_SIZE - 1;
3489
3490 /* reset all but tr, trace, and overruns */
3491 memset(&iter->seq, 0,
3492 sizeof(struct trace_iterator) -
3493 offsetof(struct trace_iterator, seq));
3494 iter->pos = -1;
3495
3496 trace_event_read_lock();
3497 trace_access_lock(iter->cpu_file);
3498 while (trace_find_next_entry_inc(iter) != NULL) {
3499 enum print_line_t ret;
3500 int len = iter->seq.len;
3501
3502 ret = print_trace_line(iter);
3503 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3504 /* don't print partial lines */
3505 iter->seq.len = len;
3506 break;
3507 }
3508 if (ret != TRACE_TYPE_NO_CONSUME)
3509 trace_consume(iter);
3510
3511 if (iter->seq.len >= cnt)
3512 break;
3513
3514 /*
3515 * Setting the full flag means we reached the trace_seq buffer
3516 * size and we should leave by partial output condition above.
3517 * One of the trace_seq_* functions is not used properly.
3518 */
3519 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3520 iter->ent->type);
3521 }
3522 trace_access_unlock(iter->cpu_file);
3523 trace_event_read_unlock();
3524
3525 /* Now copy what we have to the user */
3526 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3527 if (iter->seq.readpos >= iter->seq.len)
3528 trace_seq_init(&iter->seq);
3529
3530 /*
3531 * If there was nothing to send to user, in spite of consuming trace
3532 * entries, go back to wait for more entries.
3533 */
3534 if (sret == -EBUSY)
3535 goto waitagain;
3536
3537out:
3538 mutex_unlock(&iter->mutex);
3539
3540 return sret;
3541}
3542
3543static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3544 struct pipe_buffer *buf)
3545{
3546 __free_page(buf->page);
3547}
3548
3549static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3550 unsigned int idx)
3551{
3552 __free_page(spd->pages[idx]);
3553}
3554
3555static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3556 .can_merge = 0,
3557 .map = generic_pipe_buf_map,
3558 .unmap = generic_pipe_buf_unmap,
3559 .confirm = generic_pipe_buf_confirm,
3560 .release = tracing_pipe_buf_release,
3561 .steal = generic_pipe_buf_steal,
3562 .get = generic_pipe_buf_get,
3563};
3564
3565static size_t
3566tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3567{
3568 size_t count;
3569 int ret;
3570
3571 /* Seq buffer is page-sized, exactly what we need. */
3572 for (;;) {
3573 count = iter->seq.len;
3574 ret = print_trace_line(iter);
3575 count = iter->seq.len - count;
3576 if (rem < count) {
3577 rem = 0;
3578 iter->seq.len -= count;
3579 break;
3580 }
3581 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3582 iter->seq.len -= count;
3583 break;
3584 }
3585
3586 if (ret != TRACE_TYPE_NO_CONSUME)
3587 trace_consume(iter);
3588 rem -= count;
3589 if (!trace_find_next_entry_inc(iter)) {
3590 rem = 0;
3591 iter->ent = NULL;
3592 break;
3593 }
3594 }
3595
3596 return rem;
3597}
3598
3599static ssize_t tracing_splice_read_pipe(struct file *filp,
3600 loff_t *ppos,
3601 struct pipe_inode_info *pipe,
3602 size_t len,
3603 unsigned int flags)
3604{
3605 struct page *pages_def[PIPE_DEF_BUFFERS];
3606 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3607 struct trace_iterator *iter = filp->private_data;
3608 struct splice_pipe_desc spd = {
3609 .pages = pages_def,
3610 .partial = partial_def,
3611 .nr_pages = 0, /* This gets updated below. */
3612 .nr_pages_max = PIPE_DEF_BUFFERS,
3613 .flags = flags,
3614 .ops = &tracing_pipe_buf_ops,
3615 .spd_release = tracing_spd_release_pipe,
3616 };
3617 static struct tracer *old_tracer;
3618 ssize_t ret;
3619 size_t rem;
3620 unsigned int i;
3621
3622 if (splice_grow_spd(pipe, &spd))
3623 return -ENOMEM;
3624
3625 /* copy the tracer to avoid using a global lock all around */
3626 mutex_lock(&trace_types_lock);
3627 if (unlikely(old_tracer != current_trace && current_trace)) {
3628 old_tracer = current_trace;
3629 *iter->trace = *current_trace;
3630 }
3631 mutex_unlock(&trace_types_lock);
3632
3633 mutex_lock(&iter->mutex);
3634
3635 if (iter->trace->splice_read) {
3636 ret = iter->trace->splice_read(iter, filp,
3637 ppos, pipe, len, flags);
3638 if (ret)
3639 goto out_err;
3640 }
3641
3642 ret = tracing_wait_pipe(filp);
3643 if (ret <= 0)
3644 goto out_err;
3645
3646 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3647 ret = -EFAULT;
3648 goto out_err;
3649 }
3650
3651 trace_event_read_lock();
3652 trace_access_lock(iter->cpu_file);
3653
3654 /* Fill as many pages as possible. */
3655 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3656 spd.pages[i] = alloc_page(GFP_KERNEL);
3657 if (!spd.pages[i])
3658 break;
3659
3660 rem = tracing_fill_pipe_page(rem, iter);
3661
3662 /* Copy the data into the page, so we can start over. */
3663 ret = trace_seq_to_buffer(&iter->seq,
3664 page_address(spd.pages[i]),
3665 iter->seq.len);
3666 if (ret < 0) {
3667 __free_page(spd.pages[i]);
3668 break;
3669 }
3670 spd.partial[i].offset = 0;
3671 spd.partial[i].len = iter->seq.len;
3672
3673 trace_seq_init(&iter->seq);
3674 }
3675
3676 trace_access_unlock(iter->cpu_file);
3677 trace_event_read_unlock();
3678 mutex_unlock(&iter->mutex);
3679
3680 spd.nr_pages = i;
3681
3682 ret = splice_to_pipe(pipe, &spd);
3683out:
3684 splice_shrink_spd(&spd);
3685 return ret;
3686
3687out_err:
3688 mutex_unlock(&iter->mutex);
3689 goto out;
3690}
3691
3692struct ftrace_entries_info {
3693 struct trace_array *tr;
3694 int cpu;
3695};
3696
3697static int tracing_entries_open(struct inode *inode, struct file *filp)
3698{
3699 struct ftrace_entries_info *info;
3700
3701 if (tracing_disabled)
3702 return -ENODEV;
3703
3704 info = kzalloc(sizeof(*info), GFP_KERNEL);
3705 if (!info)
3706 return -ENOMEM;
3707
3708 info->tr = &global_trace;
3709 info->cpu = (unsigned long)inode->i_private;
3710
3711 filp->private_data = info;
3712
3713 return 0;
3714}
3715
3716static ssize_t
3717tracing_entries_read(struct file *filp, char __user *ubuf,
3718 size_t cnt, loff_t *ppos)
3719{
3720 struct ftrace_entries_info *info = filp->private_data;
3721 struct trace_array *tr = info->tr;
3722 char buf[64];
3723 int r = 0;
3724 ssize_t ret;
3725
3726 mutex_lock(&trace_types_lock);
3727
3728 if (info->cpu == RING_BUFFER_ALL_CPUS) {
3729 int cpu, buf_size_same;
3730 unsigned long size;
3731
3732 size = 0;
3733 buf_size_same = 1;
3734 /* check if all cpu sizes are same */
3735 for_each_tracing_cpu(cpu) {
3736 /* fill in the size from first enabled cpu */
3737 if (size == 0)
3738 size = tr->data[cpu]->entries;
3739 if (size != tr->data[cpu]->entries) {
3740 buf_size_same = 0;
3741 break;
3742 }
3743 }
3744
3745 if (buf_size_same) {
3746 if (!ring_buffer_expanded)
3747 r = sprintf(buf, "%lu (expanded: %lu)\n",
3748 size >> 10,
3749 trace_buf_size >> 10);
3750 else
3751 r = sprintf(buf, "%lu\n", size >> 10);
3752 } else
3753 r = sprintf(buf, "X\n");
3754 } else
3755 r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3756
3757 mutex_unlock(&trace_types_lock);
3758
3759 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3760 return ret;
3761}
3762
3763static ssize_t
3764tracing_entries_write(struct file *filp, const char __user *ubuf,
3765 size_t cnt, loff_t *ppos)
3766{
3767 struct ftrace_entries_info *info = filp->private_data;
3768 unsigned long val;
3769 int ret;
3770
3771 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3772 if (ret)
3773 return ret;
3774
3775 /* must have at least 1 entry */
3776 if (!val)
3777 return -EINVAL;
3778
3779 /* value is in KB */
3780 val <<= 10;
3781
3782 ret = tracing_resize_ring_buffer(val, info->cpu);
3783 if (ret < 0)
3784 return ret;
3785
3786 *ppos += cnt;
3787
3788 return cnt;
3789}
3790
3791static int
3792tracing_entries_release(struct inode *inode, struct file *filp)
3793{
3794 struct ftrace_entries_info *info = filp->private_data;
3795
3796 kfree(info);
3797
3798 return 0;
3799}
3800
3801static ssize_t
3802tracing_total_entries_read(struct file *filp, char __user *ubuf,
3803 size_t cnt, loff_t *ppos)
3804{
3805 struct trace_array *tr = filp->private_data;
3806 char buf[64];
3807 int r, cpu;
3808 unsigned long size = 0, expanded_size = 0;
3809
3810 mutex_lock(&trace_types_lock);
3811 for_each_tracing_cpu(cpu) {
3812 size += tr->data[cpu]->entries >> 10;
3813 if (!ring_buffer_expanded)
3814 expanded_size += trace_buf_size >> 10;
3815 }
3816 if (ring_buffer_expanded)
3817 r = sprintf(buf, "%lu\n", size);
3818 else
3819 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3820 mutex_unlock(&trace_types_lock);
3821
3822 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3823}
3824
3825static ssize_t
3826tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3827 size_t cnt, loff_t *ppos)
3828{
3829 /*
3830 * There is no need to read what the user has written, this function
3831 * is just to make sure that there is no error when "echo" is used
3832 */
3833
3834 *ppos += cnt;
3835
3836 return cnt;
3837}
3838
3839static int
3840tracing_free_buffer_release(struct inode *inode, struct file *filp)
3841{
3842 /* disable tracing ? */
3843 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3844 tracing_off();
3845 /* resize the ring buffer to 0 */
3846 tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3847
3848 return 0;
3849}
3850
3851static ssize_t
3852tracing_mark_write(struct file *filp, const char __user *ubuf,
3853 size_t cnt, loff_t *fpos)
3854{
3855 unsigned long addr = (unsigned long)ubuf;
3856 struct ring_buffer_event *event;
3857 struct ring_buffer *buffer;
3858 struct print_entry *entry;
3859 unsigned long irq_flags;
3860 struct page *pages[2];
3861 void *map_page[2];
3862 int nr_pages = 1;
3863 ssize_t written;
3864 int offset;
3865 int size;
3866 int len;
3867 int ret;
3868 int i;
3869
3870 if (tracing_disabled)
3871 return -EINVAL;
3872
3873 if (cnt > TRACE_BUF_SIZE)
3874 cnt = TRACE_BUF_SIZE;
3875
3876 /*
3877 * Userspace is injecting traces into the kernel trace buffer.
3878 * We want to be as non intrusive as possible.
3879 * To do so, we do not want to allocate any special buffers
3880 * or take any locks, but instead write the userspace data
3881 * straight into the ring buffer.
3882 *
3883 * First we need to pin the userspace buffer into memory,
3884 * which, most likely it is, because it just referenced it.
3885 * But there's no guarantee that it is. By using get_user_pages_fast()
3886 * and kmap_atomic/kunmap_atomic() we can get access to the
3887 * pages directly. We then write the data directly into the
3888 * ring buffer.
3889 */
3890 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
3891
3892 /* check if we cross pages */
3893 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3894 nr_pages = 2;
3895
3896 offset = addr & (PAGE_SIZE - 1);
3897 addr &= PAGE_MASK;
3898
3899 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3900 if (ret < nr_pages) {
3901 while (--ret >= 0)
3902 put_page(pages[ret]);
3903 written = -EFAULT;
3904 goto out;
3905 }
3906
3907 for (i = 0; i < nr_pages; i++)
3908 map_page[i] = kmap_atomic(pages[i]);
3909
3910 local_save_flags(irq_flags);
3911 size = sizeof(*entry) + cnt + 2; /* possible \n added */
3912 buffer = global_trace.buffer;
3913 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3914 irq_flags, preempt_count());
3915 if (!event) {
3916 /* Ring buffer disabled, return as if not open for write */
3917 written = -EBADF;
3918 goto out_unlock;
3919 }
3920
3921 entry = ring_buffer_event_data(event);
3922 entry->ip = _THIS_IP_;
3923
3924 if (nr_pages == 2) {
3925 len = PAGE_SIZE - offset;
3926 memcpy(&entry->buf, map_page[0] + offset, len);
3927 memcpy(&entry->buf[len], map_page[1], cnt - len);
3928 } else
3929 memcpy(&entry->buf, map_page[0] + offset, cnt);
3930
3931 if (entry->buf[cnt - 1] != '\n') {
3932 entry->buf[cnt] = '\n';
3933 entry->buf[cnt + 1] = '\0';
3934 } else
3935 entry->buf[cnt] = '\0';
3936
3937 ring_buffer_unlock_commit(buffer, event);
3938
3939 written = cnt;
3940
3941 *fpos += written;
3942
3943 out_unlock:
3944 for (i = 0; i < nr_pages; i++){
3945 kunmap_atomic(map_page[i]);
3946 put_page(pages[i]);
3947 }
3948 out:
3949 return written;
3950}
3951
3952static int tracing_clock_show(struct seq_file *m, void *v)
3953{
3954 int i;
3955
3956 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3957 seq_printf(m,
3958 "%s%s%s%s", i ? " " : "",
3959 i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3960 i == trace_clock_id ? "]" : "");
3961 seq_putc(m, '\n');
3962
3963 return 0;
3964}
3965
3966static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3967 size_t cnt, loff_t *fpos)
3968{
3969 char buf[64];
3970 const char *clockstr;
3971 int i;
3972
3973 if (cnt >= sizeof(buf))
3974 return -EINVAL;
3975
3976 if (copy_from_user(&buf, ubuf, cnt))
3977 return -EFAULT;
3978
3979 buf[cnt] = 0;
3980
3981 clockstr = strstrip(buf);
3982
3983 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3984 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3985 break;
3986 }
3987 if (i == ARRAY_SIZE(trace_clocks))
3988 return -EINVAL;
3989
3990 trace_clock_id = i;
3991
3992 mutex_lock(&trace_types_lock);
3993
3994 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3995 if (max_tr.buffer)
3996 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3997
3998 mutex_unlock(&trace_types_lock);
3999
4000 *fpos += cnt;
4001
4002 return cnt;
4003}
4004
4005static int tracing_clock_open(struct inode *inode, struct file *file)
4006{
4007 if (tracing_disabled)
4008 return -ENODEV;
4009 return single_open(file, tracing_clock_show, NULL);
4010}
4011
4012static const struct file_operations tracing_max_lat_fops = {
4013 .open = tracing_open_generic,
4014 .read = tracing_max_lat_read,
4015 .write = tracing_max_lat_write,
4016 .llseek = generic_file_llseek,
4017};
4018
4019static const struct file_operations tracing_ctrl_fops = {
4020 .open = tracing_open_generic,
4021 .read = tracing_ctrl_read,
4022 .write = tracing_ctrl_write,
4023 .llseek = generic_file_llseek,
4024};
4025
4026static const struct file_operations set_tracer_fops = {
4027 .open = tracing_open_generic,
4028 .read = tracing_set_trace_read,
4029 .write = tracing_set_trace_write,
4030 .llseek = generic_file_llseek,
4031};
4032
4033static const struct file_operations tracing_pipe_fops = {
4034 .open = tracing_open_pipe,
4035 .poll = tracing_poll_pipe,
4036 .read = tracing_read_pipe,
4037 .splice_read = tracing_splice_read_pipe,
4038 .release = tracing_release_pipe,
4039 .llseek = no_llseek,
4040};
4041
4042static const struct file_operations tracing_entries_fops = {
4043 .open = tracing_entries_open,
4044 .read = tracing_entries_read,
4045 .write = tracing_entries_write,
4046 .release = tracing_entries_release,
4047 .llseek = generic_file_llseek,
4048};
4049
4050static const struct file_operations tracing_total_entries_fops = {
4051 .open = tracing_open_generic,
4052 .read = tracing_total_entries_read,
4053 .llseek = generic_file_llseek,
4054};
4055
4056static const struct file_operations tracing_free_buffer_fops = {
4057 .write = tracing_free_buffer_write,
4058 .release = tracing_free_buffer_release,
4059};
4060
4061static const struct file_operations tracing_mark_fops = {
4062 .open = tracing_open_generic,
4063 .write = tracing_mark_write,
4064 .llseek = generic_file_llseek,
4065};
4066
4067static const struct file_operations trace_clock_fops = {
4068 .open = tracing_clock_open,
4069 .read = seq_read,
4070 .llseek = seq_lseek,
4071 .release = single_release,
4072 .write = tracing_clock_write,
4073};
4074
4075struct ftrace_buffer_info {
4076 struct trace_array *tr;
4077 void *spare;
4078 int cpu;
4079 unsigned int read;
4080};
4081
4082static int tracing_buffers_open(struct inode *inode, struct file *filp)
4083{
4084 int cpu = (int)(long)inode->i_private;
4085 struct ftrace_buffer_info *info;
4086
4087 if (tracing_disabled)
4088 return -ENODEV;
4089
4090 info = kzalloc(sizeof(*info), GFP_KERNEL);
4091 if (!info)
4092 return -ENOMEM;
4093
4094 info->tr = &global_trace;
4095 info->cpu = cpu;
4096 info->spare = NULL;
4097 /* Force reading ring buffer for first read */
4098 info->read = (unsigned int)-1;
4099
4100 filp->private_data = info;
4101
4102 return nonseekable_open(inode, filp);
4103}
4104
4105static ssize_t
4106tracing_buffers_read(struct file *filp, char __user *ubuf,
4107 size_t count, loff_t *ppos)
4108{
4109 struct ftrace_buffer_info *info = filp->private_data;
4110 ssize_t ret;
4111 size_t size;
4112
4113 if (!count)
4114 return 0;
4115
4116 if (!info->spare)
4117 info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4118 if (!info->spare)
4119 return -ENOMEM;
4120
4121 /* Do we have previous read data to read? */
4122 if (info->read < PAGE_SIZE)
4123 goto read;
4124
4125 trace_access_lock(info->cpu);
4126 ret = ring_buffer_read_page(info->tr->buffer,
4127 &info->spare,
4128 count,
4129 info->cpu, 0);
4130 trace_access_unlock(info->cpu);
4131 if (ret < 0)
4132 return 0;
4133
4134 info->read = 0;
4135
4136read:
4137 size = PAGE_SIZE - info->read;
4138 if (size > count)
4139 size = count;
4140
4141 ret = copy_to_user(ubuf, info->spare + info->read, size);
4142 if (ret == size)
4143 return -EFAULT;
4144 size -= ret;
4145
4146 *ppos += size;
4147 info->read += size;
4148
4149 return size;
4150}
4151
4152static int tracing_buffers_release(struct inode *inode, struct file *file)
4153{
4154 struct ftrace_buffer_info *info = file->private_data;
4155
4156 if (info->spare)
4157 ring_buffer_free_read_page(info->tr->buffer, info->spare);
4158 kfree(info);
4159
4160 return 0;
4161}
4162
4163struct buffer_ref {
4164 struct ring_buffer *buffer;
4165 void *page;
4166 int ref;
4167};
4168
4169static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4170 struct pipe_buffer *buf)
4171{
4172 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4173
4174 if (--ref->ref)
4175 return;
4176
4177 ring_buffer_free_read_page(ref->buffer, ref->page);
4178 kfree(ref);
4179 buf->private = 0;
4180}
4181
4182static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
4183 struct pipe_buffer *buf)
4184{
4185 return 1;
4186}
4187
4188static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4189 struct pipe_buffer *buf)
4190{
4191 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4192
4193 ref->ref++;
4194}
4195
4196/* Pipe buffer operations for a buffer. */
4197static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4198 .can_merge = 0,
4199 .map = generic_pipe_buf_map,
4200 .unmap = generic_pipe_buf_unmap,
4201 .confirm = generic_pipe_buf_confirm,
4202 .release = buffer_pipe_buf_release,
4203 .steal = buffer_pipe_buf_steal,
4204 .get = buffer_pipe_buf_get,
4205};
4206
4207/*
4208 * Callback from splice_to_pipe(), if we need to release some pages
4209 * at the end of the spd in case we error'ed out in filling the pipe.
4210 */
4211static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4212{
4213 struct buffer_ref *ref =
4214 (struct buffer_ref *)spd->partial[i].private;
4215
4216 if (--ref->ref)
4217 return;
4218
4219 ring_buffer_free_read_page(ref->buffer, ref->page);
4220 kfree(ref);
4221 spd->partial[i].private = 0;
4222}
4223
4224static ssize_t
4225tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4226 struct pipe_inode_info *pipe, size_t len,
4227 unsigned int flags)
4228{
4229 struct ftrace_buffer_info *info = file->private_data;
4230 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4231 struct page *pages_def[PIPE_DEF_BUFFERS];
4232 struct splice_pipe_desc spd = {
4233 .pages = pages_def,
4234 .partial = partial_def,
4235 .nr_pages_max = PIPE_DEF_BUFFERS,
4236 .flags = flags,
4237 .ops = &buffer_pipe_buf_ops,
4238 .spd_release = buffer_spd_release,
4239 };
4240 struct buffer_ref *ref;
4241 int entries, size, i;
4242 size_t ret;
4243
4244 if (splice_grow_spd(pipe, &spd))
4245 return -ENOMEM;
4246
4247 if (*ppos & (PAGE_SIZE - 1)) {
4248 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
4249 ret = -EINVAL;
4250 goto out;
4251 }
4252
4253 if (len & (PAGE_SIZE - 1)) {
4254 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
4255 if (len < PAGE_SIZE) {
4256 ret = -EINVAL;
4257 goto out;
4258 }
4259 len &= PAGE_MASK;
4260 }
4261
4262 trace_access_lock(info->cpu);
4263 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4264
4265 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4266 struct page *page;
4267 int r;
4268
4269 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4270 if (!ref)
4271 break;
4272
4273 ref->ref = 1;
4274 ref->buffer = info->tr->buffer;
4275 ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4276 if (!ref->page) {
4277 kfree(ref);
4278 break;
4279 }
4280
4281 r = ring_buffer_read_page(ref->buffer, &ref->page,
4282 len, info->cpu, 1);
4283 if (r < 0) {
4284 ring_buffer_free_read_page(ref->buffer, ref->page);
4285 kfree(ref);
4286 break;
4287 }
4288
4289 /*
4290 * zero out any left over data, this is going to
4291 * user land.
4292 */
4293 size = ring_buffer_page_len(ref->page);
4294 if (size < PAGE_SIZE)
4295 memset(ref->page + size, 0, PAGE_SIZE - size);
4296
4297 page = virt_to_page(ref->page);
4298
4299 spd.pages[i] = page;
4300 spd.partial[i].len = PAGE_SIZE;
4301 spd.partial[i].offset = 0;
4302 spd.partial[i].private = (unsigned long)ref;
4303 spd.nr_pages++;
4304 *ppos += PAGE_SIZE;
4305
4306 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4307 }
4308
4309 trace_access_unlock(info->cpu);
4310 spd.nr_pages = i;
4311
4312 /* did we read anything? */
4313 if (!spd.nr_pages) {
4314 if (flags & SPLICE_F_NONBLOCK)
4315 ret = -EAGAIN;
4316 else
4317 ret = 0;
4318 /* TODO: block */
4319 goto out;
4320 }
4321
4322 ret = splice_to_pipe(pipe, &spd);
4323 splice_shrink_spd(&spd);
4324out:
4325 return ret;
4326}
4327
4328static const struct file_operations tracing_buffers_fops = {
4329 .open = tracing_buffers_open,
4330 .read = tracing_buffers_read,
4331 .release = tracing_buffers_release,
4332 .splice_read = tracing_buffers_splice_read,
4333 .llseek = no_llseek,
4334};
4335
4336static ssize_t
4337tracing_stats_read(struct file *filp, char __user *ubuf,
4338 size_t count, loff_t *ppos)
4339{
4340 unsigned long cpu = (unsigned long)filp->private_data;
4341 struct trace_array *tr = &global_trace;
4342 struct trace_seq *s;
4343 unsigned long cnt;
4344 unsigned long long t;
4345 unsigned long usec_rem;
4346
4347 s = kmalloc(sizeof(*s), GFP_KERNEL);
4348 if (!s)
4349 return -ENOMEM;
4350
4351 trace_seq_init(s);
4352
4353 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4354 trace_seq_printf(s, "entries: %ld\n", cnt);
4355
4356 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4357 trace_seq_printf(s, "overrun: %ld\n", cnt);
4358
4359 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4360 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4361
4362 cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4363 trace_seq_printf(s, "bytes: %ld\n", cnt);
4364
4365 t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4366 usec_rem = do_div(t, USEC_PER_SEC);
4367 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
4368
4369 t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4370 usec_rem = do_div(t, USEC_PER_SEC);
4371 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4372
4373 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4374
4375 kfree(s);
4376
4377 return count;
4378}
4379
4380static const struct file_operations tracing_stats_fops = {
4381 .open = tracing_open_generic,
4382 .read = tracing_stats_read,
4383 .llseek = generic_file_llseek,
4384};
4385
4386#ifdef CONFIG_DYNAMIC_FTRACE
4387
4388int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4389{
4390 return 0;
4391}
4392
4393static ssize_t
4394tracing_read_dyn_info(struct file *filp, char __user *ubuf,
4395 size_t cnt, loff_t *ppos)
4396{
4397 static char ftrace_dyn_info_buffer[1024];
4398 static DEFINE_MUTEX(dyn_info_mutex);
4399 unsigned long *p = filp->private_data;
4400 char *buf = ftrace_dyn_info_buffer;
4401 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4402 int r;
4403
4404 mutex_lock(&dyn_info_mutex);
4405 r = sprintf(buf, "%ld ", *p);
4406
4407 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4408 buf[r++] = '\n';
4409
4410 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4411
4412 mutex_unlock(&dyn_info_mutex);
4413
4414 return r;
4415}
4416
4417static const struct file_operations tracing_dyn_info_fops = {
4418 .open = tracing_open_generic,
4419 .read = tracing_read_dyn_info,
4420 .llseek = generic_file_llseek,
4421};
4422#endif
4423
4424static struct dentry *d_tracer;
4425
4426struct dentry *tracing_init_dentry(void)
4427{
4428 static int once;
4429
4430 if (d_tracer)
4431 return d_tracer;
4432
4433 if (!debugfs_initialized())
4434 return NULL;
4435
4436 d_tracer = debugfs_create_dir("tracing", NULL);
4437
4438 if (!d_tracer && !once) {
4439 once = 1;
4440 pr_warning("Could not create debugfs directory 'tracing'\n");
4441 return NULL;
4442 }
4443
4444 return d_tracer;
4445}
4446
4447static struct dentry *d_percpu;
4448
4449struct dentry *tracing_dentry_percpu(void)
4450{
4451 static int once;
4452 struct dentry *d_tracer;
4453
4454 if (d_percpu)
4455 return d_percpu;
4456
4457 d_tracer = tracing_init_dentry();
4458
4459 if (!d_tracer)
4460 return NULL;
4461
4462 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4463
4464 if (!d_percpu && !once) {
4465 once = 1;
4466 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4467 return NULL;
4468 }
4469
4470 return d_percpu;
4471}
4472
4473static void tracing_init_debugfs_percpu(long cpu)
4474{
4475 struct dentry *d_percpu = tracing_dentry_percpu();
4476 struct dentry *d_cpu;
4477 char cpu_dir[30]; /* 30 characters should be more than enough */
4478
4479 if (!d_percpu)
4480 return;
4481
4482 snprintf(cpu_dir, 30, "cpu%ld", cpu);
4483 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4484 if (!d_cpu) {
4485 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4486 return;
4487 }
4488
4489 /* per cpu trace_pipe */
4490 trace_create_file("trace_pipe", 0444, d_cpu,
4491 (void *) cpu, &tracing_pipe_fops);
4492
4493 /* per cpu trace */
4494 trace_create_file("trace", 0644, d_cpu,
4495 (void *) cpu, &tracing_fops);
4496
4497 trace_create_file("trace_pipe_raw", 0444, d_cpu,
4498 (void *) cpu, &tracing_buffers_fops);
4499
4500 trace_create_file("stats", 0444, d_cpu,
4501 (void *) cpu, &tracing_stats_fops);
4502
4503 trace_create_file("buffer_size_kb", 0444, d_cpu,
4504 (void *) cpu, &tracing_entries_fops);
4505}
4506
4507#ifdef CONFIG_FTRACE_SELFTEST
4508/* Let selftest have access to static functions in this file */
4509#include "trace_selftest.c"
4510#endif
4511
4512struct trace_option_dentry {
4513 struct tracer_opt *opt;
4514 struct tracer_flags *flags;
4515 struct dentry *entry;
4516};
4517
4518static ssize_t
4519trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4520 loff_t *ppos)
4521{
4522 struct trace_option_dentry *topt = filp->private_data;
4523 char *buf;
4524
4525 if (topt->flags->val & topt->opt->bit)
4526 buf = "1\n";
4527 else
4528 buf = "0\n";
4529
4530 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4531}
4532
4533static ssize_t
4534trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4535 loff_t *ppos)
4536{
4537 struct trace_option_dentry *topt = filp->private_data;
4538 unsigned long val;
4539 int ret;
4540
4541 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4542 if (ret)
4543 return ret;
4544
4545 if (val != 0 && val != 1)
4546 return -EINVAL;
4547
4548 if (!!(topt->flags->val & topt->opt->bit) != val) {
4549 mutex_lock(&trace_types_lock);
4550 ret = __set_tracer_option(current_trace, topt->flags,
4551 topt->opt, !val);
4552 mutex_unlock(&trace_types_lock);
4553 if (ret)
4554 return ret;
4555 }
4556
4557 *ppos += cnt;
4558
4559 return cnt;
4560}
4561
4562
4563static const struct file_operations trace_options_fops = {
4564 .open = tracing_open_generic,
4565 .read = trace_options_read,
4566 .write = trace_options_write,
4567 .llseek = generic_file_llseek,
4568};
4569
4570static ssize_t
4571trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4572 loff_t *ppos)
4573{
4574 long index = (long)filp->private_data;
4575 char *buf;
4576
4577 if (trace_flags & (1 << index))
4578 buf = "1\n";
4579 else
4580 buf = "0\n";
4581
4582 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4583}
4584
4585static ssize_t
4586trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4587 loff_t *ppos)
4588{
4589 long index = (long)filp->private_data;
4590 unsigned long val;
4591 int ret;
4592
4593 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4594 if (ret)
4595 return ret;
4596
4597 if (val != 0 && val != 1)
4598 return -EINVAL;
4599 set_tracer_flags(1 << index, val);
4600
4601 *ppos += cnt;
4602
4603 return cnt;
4604}
4605
4606static const struct file_operations trace_options_core_fops = {
4607 .open = tracing_open_generic,
4608 .read = trace_options_core_read,
4609 .write = trace_options_core_write,
4610 .llseek = generic_file_llseek,
4611};
4612
4613struct dentry *trace_create_file(const char *name,
4614 umode_t mode,
4615 struct dentry *parent,
4616 void *data,
4617 const struct file_operations *fops)
4618{
4619 struct dentry *ret;
4620
4621 ret = debugfs_create_file(name, mode, parent, data, fops);
4622 if (!ret)
4623 pr_warning("Could not create debugfs '%s' entry\n", name);
4624
4625 return ret;
4626}
4627
4628
4629static struct dentry *trace_options_init_dentry(void)
4630{
4631 struct dentry *d_tracer;
4632 static struct dentry *t_options;
4633
4634 if (t_options)
4635 return t_options;
4636
4637 d_tracer = tracing_init_dentry();
4638 if (!d_tracer)
4639 return NULL;
4640
4641 t_options = debugfs_create_dir("options", d_tracer);
4642 if (!t_options) {
4643 pr_warning("Could not create debugfs directory 'options'\n");
4644 return NULL;
4645 }
4646
4647 return t_options;
4648}
4649
4650static void
4651create_trace_option_file(struct trace_option_dentry *topt,
4652 struct tracer_flags *flags,
4653 struct tracer_opt *opt)
4654{
4655 struct dentry *t_options;
4656
4657 t_options = trace_options_init_dentry();
4658 if (!t_options)
4659 return;
4660
4661 topt->flags = flags;
4662 topt->opt = opt;
4663
4664 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4665 &trace_options_fops);
4666
4667}
4668
4669static struct trace_option_dentry *
4670create_trace_option_files(struct tracer *tracer)
4671{
4672 struct trace_option_dentry *topts;
4673 struct tracer_flags *flags;
4674 struct tracer_opt *opts;
4675 int cnt;
4676
4677 if (!tracer)
4678 return NULL;
4679
4680 flags = tracer->flags;
4681
4682 if (!flags || !flags->opts)
4683 return NULL;
4684
4685 opts = flags->opts;
4686
4687 for (cnt = 0; opts[cnt].name; cnt++)
4688 ;
4689
4690 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4691 if (!topts)
4692 return NULL;
4693
4694 for (cnt = 0; opts[cnt].name; cnt++)
4695 create_trace_option_file(&topts[cnt], flags,
4696 &opts[cnt]);
4697
4698 return topts;
4699}
4700
4701static void
4702destroy_trace_option_files(struct trace_option_dentry *topts)
4703{
4704 int cnt;
4705
4706 if (!topts)
4707 return;
4708
4709 for (cnt = 0; topts[cnt].opt; cnt++) {
4710 if (topts[cnt].entry)
4711 debugfs_remove(topts[cnt].entry);
4712 }
4713
4714 kfree(topts);
4715}
4716
4717static struct dentry *
4718create_trace_option_core_file(const char *option, long index)
4719{
4720 struct dentry *t_options;
4721
4722 t_options = trace_options_init_dentry();
4723 if (!t_options)
4724 return NULL;
4725
4726 return trace_create_file(option, 0644, t_options, (void *)index,
4727 &trace_options_core_fops);
4728}
4729
4730static __init void create_trace_options_dir(void)
4731{
4732 struct dentry *t_options;
4733 int i;
4734
4735 t_options = trace_options_init_dentry();
4736 if (!t_options)
4737 return;
4738
4739 for (i = 0; trace_options[i]; i++)
4740 create_trace_option_core_file(trace_options[i], i);
4741}
4742
4743static ssize_t
4744rb_simple_read(struct file *filp, char __user *ubuf,
4745 size_t cnt, loff_t *ppos)
4746{
4747 struct trace_array *tr = filp->private_data;
4748 struct ring_buffer *buffer = tr->buffer;
4749 char buf[64];
4750 int r;
4751
4752 if (buffer)
4753 r = ring_buffer_record_is_on(buffer);
4754 else
4755 r = 0;
4756
4757 r = sprintf(buf, "%d\n", r);
4758
4759 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4760}
4761
4762static ssize_t
4763rb_simple_write(struct file *filp, const char __user *ubuf,
4764 size_t cnt, loff_t *ppos)
4765{
4766 struct trace_array *tr = filp->private_data;
4767 struct ring_buffer *buffer = tr->buffer;
4768 unsigned long val;
4769 int ret;
4770
4771 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4772 if (ret)
4773 return ret;
4774
4775 if (buffer) {
4776 if (val)
4777 ring_buffer_record_on(buffer);
4778 else
4779 ring_buffer_record_off(buffer);
4780 }
4781
4782 (*ppos)++;
4783
4784 return cnt;
4785}
4786
4787static const struct file_operations rb_simple_fops = {
4788 .open = tracing_open_generic,
4789 .read = rb_simple_read,
4790 .write = rb_simple_write,
4791 .llseek = default_llseek,
4792};
4793
4794static __init int tracer_init_debugfs(void)
4795{
4796 struct dentry *d_tracer;
4797 int cpu;
4798
4799 trace_access_lock_init();
4800
4801 d_tracer = tracing_init_dentry();
4802
4803 trace_create_file("tracing_enabled", 0644, d_tracer,
4804 &global_trace, &tracing_ctrl_fops);
4805
4806 trace_create_file("trace_options", 0644, d_tracer,
4807 NULL, &tracing_iter_fops);
4808
4809 trace_create_file("tracing_cpumask", 0644, d_tracer,
4810 NULL, &tracing_cpumask_fops);
4811
4812 trace_create_file("trace", 0644, d_tracer,
4813 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4814
4815 trace_create_file("available_tracers", 0444, d_tracer,
4816 &global_trace, &show_traces_fops);
4817
4818 trace_create_file("current_tracer", 0644, d_tracer,
4819 &global_trace, &set_tracer_fops);
4820
4821#ifdef CONFIG_TRACER_MAX_TRACE
4822 trace_create_file("tracing_max_latency", 0644, d_tracer,
4823 &tracing_max_latency, &tracing_max_lat_fops);
4824#endif
4825
4826 trace_create_file("tracing_thresh", 0644, d_tracer,
4827 &tracing_thresh, &tracing_max_lat_fops);
4828
4829 trace_create_file("README", 0444, d_tracer,
4830 NULL, &tracing_readme_fops);
4831
4832 trace_create_file("trace_pipe", 0444, d_tracer,
4833 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4834
4835 trace_create_file("buffer_size_kb", 0644, d_tracer,
4836 (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4837
4838 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4839 &global_trace, &tracing_total_entries_fops);
4840
4841 trace_create_file("free_buffer", 0644, d_tracer,
4842 &global_trace, &tracing_free_buffer_fops);
4843
4844 trace_create_file("trace_marker", 0220, d_tracer,
4845 NULL, &tracing_mark_fops);
4846
4847 trace_create_file("saved_cmdlines", 0444, d_tracer,
4848 NULL, &tracing_saved_cmdlines_fops);
4849
4850 trace_create_file("trace_clock", 0644, d_tracer, NULL,
4851 &trace_clock_fops);
4852
4853 trace_create_file("tracing_on", 0644, d_tracer,
4854 &global_trace, &rb_simple_fops);
4855
4856#ifdef CONFIG_DYNAMIC_FTRACE
4857 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4858 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4859#endif
4860
4861 create_trace_options_dir();
4862
4863 for_each_tracing_cpu(cpu)
4864 tracing_init_debugfs_percpu(cpu);
4865
4866 return 0;
4867}
4868
4869static int trace_panic_handler(struct notifier_block *this,
4870 unsigned long event, void *unused)
4871{
4872 if (ftrace_dump_on_oops)
4873 ftrace_dump(ftrace_dump_on_oops);
4874 return NOTIFY_OK;
4875}
4876
4877static struct notifier_block trace_panic_notifier = {
4878 .notifier_call = trace_panic_handler,
4879 .next = NULL,
4880 .priority = 150 /* priority: INT_MAX >= x >= 0 */
4881};
4882
4883static int trace_die_handler(struct notifier_block *self,
4884 unsigned long val,
4885 void *data)
4886{
4887 switch (val) {
4888 case DIE_OOPS:
4889 if (ftrace_dump_on_oops)
4890 ftrace_dump(ftrace_dump_on_oops);
4891 break;
4892 default:
4893 break;
4894 }
4895 return NOTIFY_OK;
4896}
4897
4898static struct notifier_block trace_die_notifier = {
4899 .notifier_call = trace_die_handler,
4900 .priority = 200
4901};
4902
4903/*
4904 * printk is set to max of 1024, we really don't need it that big.
4905 * Nothing should be printing 1000 characters anyway.
4906 */
4907#define TRACE_MAX_PRINT 1000
4908
4909/*
4910 * Define here KERN_TRACE so that we have one place to modify
4911 * it if we decide to change what log level the ftrace dump
4912 * should be at.
4913 */
4914#define KERN_TRACE KERN_EMERG
4915
4916void
4917trace_printk_seq(struct trace_seq *s)
4918{
4919 /* Probably should print a warning here. */
4920 if (s->len >= 1000)
4921 s->len = 1000;
4922
4923 /* should be zero ended, but we are paranoid. */
4924 s->buffer[s->len] = 0;
4925
4926 printk(KERN_TRACE "%s", s->buffer);
4927
4928 trace_seq_init(s);
4929}
4930
4931void trace_init_global_iter(struct trace_iterator *iter)
4932{
4933 iter->tr = &global_trace;
4934 iter->trace = current_trace;
4935 iter->cpu_file = TRACE_PIPE_ALL_CPU;
4936}
4937
4938static void
4939__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4940{
4941 static arch_spinlock_t ftrace_dump_lock =
4942 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4943 /* use static because iter can be a bit big for the stack */
4944 static struct trace_iterator iter;
4945 unsigned int old_userobj;
4946 static int dump_ran;
4947 unsigned long flags;
4948 int cnt = 0, cpu;
4949
4950 /* only one dump */
4951 local_irq_save(flags);
4952 arch_spin_lock(&ftrace_dump_lock);
4953 if (dump_ran)
4954 goto out;
4955
4956 dump_ran = 1;
4957
4958 tracing_off();
4959
4960 /* Did function tracer already get disabled? */
4961 if (ftrace_is_dead()) {
4962 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
4963 printk("# MAY BE MISSING FUNCTION EVENTS\n");
4964 }
4965
4966 if (disable_tracing)
4967 ftrace_kill();
4968
4969 trace_init_global_iter(&iter);
4970
4971 for_each_tracing_cpu(cpu) {
4972 atomic_inc(&iter.tr->data[cpu]->disabled);
4973 }
4974
4975 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4976
4977 /* don't look at user memory in panic mode */
4978 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4979
4980 /* Simulate the iterator */
4981 iter.tr = &global_trace;
4982 iter.trace = current_trace;
4983
4984 switch (oops_dump_mode) {
4985 case DUMP_ALL:
4986 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4987 break;
4988 case DUMP_ORIG:
4989 iter.cpu_file = raw_smp_processor_id();
4990 break;
4991 case DUMP_NONE:
4992 goto out_enable;
4993 default:
4994 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4995 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4996 }
4997
4998 printk(KERN_TRACE "Dumping ftrace buffer:\n");
4999
5000 /*
5001 * We need to stop all tracing on all CPUS to read the
5002 * the next buffer. This is a bit expensive, but is
5003 * not done often. We fill all what we can read,
5004 * and then release the locks again.
5005 */
5006
5007 while (!trace_empty(&iter)) {
5008
5009 if (!cnt)
5010 printk(KERN_TRACE "---------------------------------\n");
5011
5012 cnt++;
5013
5014 /* reset all but tr, trace, and overruns */
5015 memset(&iter.seq, 0,
5016 sizeof(struct trace_iterator) -
5017 offsetof(struct trace_iterator, seq));
5018 iter.iter_flags |= TRACE_FILE_LAT_FMT;
5019 iter.pos = -1;
5020
5021 if (trace_find_next_entry_inc(&iter) != NULL) {
5022 int ret;
5023
5024 ret = print_trace_line(&iter);
5025 if (ret != TRACE_TYPE_NO_CONSUME)
5026 trace_consume(&iter);
5027 }
5028 touch_nmi_watchdog();
5029
5030 trace_printk_seq(&iter.seq);
5031 }
5032
5033 if (!cnt)
5034 printk(KERN_TRACE " (ftrace buffer empty)\n");
5035 else
5036 printk(KERN_TRACE "---------------------------------\n");
5037
5038 out_enable:
5039 /* Re-enable tracing if requested */
5040 if (!disable_tracing) {
5041 trace_flags |= old_userobj;
5042
5043 for_each_tracing_cpu(cpu) {
5044 atomic_dec(&iter.tr->data[cpu]->disabled);
5045 }
5046 tracing_on();
5047 }
5048
5049 out:
5050 arch_spin_unlock(&ftrace_dump_lock);
5051 local_irq_restore(flags);
5052}
5053
5054/* By default: disable tracing after the dump */
5055void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5056{
5057 __ftrace_dump(true, oops_dump_mode);
5058}
5059EXPORT_SYMBOL_GPL(ftrace_dump);
5060
5061__init static int tracer_alloc_buffers(void)
5062{
5063 int ring_buf_size;
5064 enum ring_buffer_flags rb_flags;
5065 int i;
5066 int ret = -ENOMEM;
5067
5068
5069 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5070 goto out;
5071
5072 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5073 goto out_free_buffer_mask;
5074
5075 /* Only allocate trace_printk buffers if a trace_printk exists */
5076 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5077 trace_printk_init_buffers();
5078
5079 /* To save memory, keep the ring buffer size to its minimum */
5080 if (ring_buffer_expanded)
5081 ring_buf_size = trace_buf_size;
5082 else
5083 ring_buf_size = 1;
5084
5085 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5086
5087 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5088 cpumask_copy(tracing_cpumask, cpu_all_mask);
5089
5090 /* TODO: make the number of buffers hot pluggable with CPUS */
5091 global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5092 if (!global_trace.buffer) {
5093 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5094 WARN_ON(1);
5095 goto out_free_cpumask;
5096 }
5097 if (global_trace.buffer_disabled)
5098 tracing_off();
5099
5100
5101#ifdef CONFIG_TRACER_MAX_TRACE
5102 max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5103 if (!max_tr.buffer) {
5104 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5105 WARN_ON(1);
5106 ring_buffer_free(global_trace.buffer);
5107 goto out_free_cpumask;
5108 }
5109#endif
5110
5111 /* Allocate the first page for all buffers */
5112 for_each_tracing_cpu(i) {
5113 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
5114 max_tr.data[i] = &per_cpu(max_tr_data, i);
5115 }
5116
5117 set_buffer_entries(&global_trace,
5118 ring_buffer_size(global_trace.buffer, 0));
5119#ifdef CONFIG_TRACER_MAX_TRACE
5120 set_buffer_entries(&max_tr, 1);
5121#endif
5122
5123 trace_init_cmdlines();
5124
5125 register_tracer(&nop_trace);
5126 current_trace = &nop_trace;
5127 /* All seems OK, enable tracing */
5128 tracing_disabled = 0;
5129
5130 atomic_notifier_chain_register(&panic_notifier_list,
5131 &trace_panic_notifier);
5132
5133 register_die_notifier(&trace_die_notifier);
5134
5135 return 0;
5136
5137out_free_cpumask:
5138 free_cpumask_var(tracing_cpumask);
5139out_free_buffer_mask:
5140 free_cpumask_var(tracing_buffer_mask);
5141out:
5142 return ret;
5143}
5144
5145__init static int clear_boot_tracer(void)
5146{
5147 /*
5148 * The default tracer at boot buffer is an init section.
5149 * This function is called in lateinit. If we did not
5150 * find the boot tracer, then clear it out, to prevent
5151 * later registration from accessing the buffer that is
5152 * about to be freed.
5153 */
5154 if (!default_bootup_tracer)
5155 return 0;
5156
5157 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5158 default_bootup_tracer);
5159 default_bootup_tracer = NULL;
5160
5161 return 0;
5162}
5163
5164early_initcall(tracer_alloc_buffers);
5165fs_initcall(tracer_init_debugfs);
5166late_initcall(clear_boot_tracer);
1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
14#include <linux/ring_buffer.h>
15#include <generated/utsrelease.h>
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
20#include <linux/notifier.h>
21#include <linux/irqflags.h>
22#include <linux/debugfs.h>
23#include <linux/pagemap.h>
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
27#include <linux/kprobes.h>
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
31#include <linux/splice.h>
32#include <linux/kdebug.h>
33#include <linux/string.h>
34#include <linux/rwsem.h>
35#include <linux/slab.h>
36#include <linux/ctype.h>
37#include <linux/init.h>
38#include <linux/poll.h>
39#include <linux/fs.h>
40
41#include "trace.h"
42#include "trace_output.h"
43
44/*
45 * On boot up, the ring buffer is set to the minimum size, so that
46 * we do not waste memory on systems that are not using tracing.
47 */
48int ring_buffer_expanded;
49
50/*
51 * We need to change this state when a selftest is running.
52 * A selftest will lurk into the ring-buffer to count the
53 * entries inserted during the selftest although some concurrent
54 * insertions into the ring-buffer such as trace_printk could occurred
55 * at the same time, giving false positive or negative results.
56 */
57static bool __read_mostly tracing_selftest_running;
58
59/*
60 * If a tracer is running, we do not want to run SELFTEST.
61 */
62bool __read_mostly tracing_selftest_disabled;
63
64/* For tracers that don't implement custom flags */
65static struct tracer_opt dummy_tracer_opt[] = {
66 { }
67};
68
69static struct tracer_flags dummy_tracer_flags = {
70 .val = 0,
71 .opts = dummy_tracer_opt
72};
73
74static int dummy_set_flag(u32 old_flags, u32 bit, int set)
75{
76 return 0;
77}
78
79/*
80 * Kill all tracing for good (never come back).
81 * It is initialized to 1 but will turn to zero if the initialization
82 * of the tracer is successful. But that is the only place that sets
83 * this back to zero.
84 */
85static int tracing_disabled = 1;
86
87DEFINE_PER_CPU(int, ftrace_cpu_disabled);
88
89static inline void ftrace_disable_cpu(void)
90{
91 preempt_disable();
92 __this_cpu_inc(ftrace_cpu_disabled);
93}
94
95static inline void ftrace_enable_cpu(void)
96{
97 __this_cpu_dec(ftrace_cpu_disabled);
98 preempt_enable();
99}
100
101cpumask_var_t __read_mostly tracing_buffer_mask;
102
103/*
104 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
105 *
106 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
107 * is set, then ftrace_dump is called. This will output the contents
108 * of the ftrace buffers to the console. This is very useful for
109 * capturing traces that lead to crashes and outputing it to a
110 * serial console.
111 *
112 * It is default off, but you can enable it with either specifying
113 * "ftrace_dump_on_oops" in the kernel command line, or setting
114 * /proc/sys/kernel/ftrace_dump_on_oops
115 * Set 1 if you want to dump buffers of all CPUs
116 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 */
118
119enum ftrace_dump_mode ftrace_dump_on_oops;
120
121static int tracing_set_tracer(const char *buf);
122
123#define MAX_TRACER_SIZE 100
124static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
125static char *default_bootup_tracer;
126
127static int __init set_cmdline_ftrace(char *str)
128{
129 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
130 default_bootup_tracer = bootup_tracer_buf;
131 /* We are using ftrace early, expand it */
132 ring_buffer_expanded = 1;
133 return 1;
134}
135__setup("ftrace=", set_cmdline_ftrace);
136
137static int __init set_ftrace_dump_on_oops(char *str)
138{
139 if (*str++ != '=' || !*str) {
140 ftrace_dump_on_oops = DUMP_ALL;
141 return 1;
142 }
143
144 if (!strcmp("orig_cpu", str)) {
145 ftrace_dump_on_oops = DUMP_ORIG;
146 return 1;
147 }
148
149 return 0;
150}
151__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
152
153unsigned long long ns2usecs(cycle_t nsec)
154{
155 nsec += 500;
156 do_div(nsec, 1000);
157 return nsec;
158}
159
160/*
161 * The global_trace is the descriptor that holds the tracing
162 * buffers for the live tracing. For each CPU, it contains
163 * a link list of pages that will store trace entries. The
164 * page descriptor of the pages in the memory is used to hold
165 * the link list by linking the lru item in the page descriptor
166 * to each of the pages in the buffer per CPU.
167 *
168 * For each active CPU there is a data field that holds the
169 * pages for the buffer for that CPU. Each CPU has the same number
170 * of pages allocated for its buffer.
171 */
172static struct trace_array global_trace;
173
174static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
175
176int filter_current_check_discard(struct ring_buffer *buffer,
177 struct ftrace_event_call *call, void *rec,
178 struct ring_buffer_event *event)
179{
180 return filter_check_discard(call, rec, buffer, event);
181}
182EXPORT_SYMBOL_GPL(filter_current_check_discard);
183
184cycle_t ftrace_now(int cpu)
185{
186 u64 ts;
187
188 /* Early boot up does not have a buffer yet */
189 if (!global_trace.buffer)
190 return trace_clock_local();
191
192 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
193 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
194
195 return ts;
196}
197
198/*
199 * The max_tr is used to snapshot the global_trace when a maximum
200 * latency is reached. Some tracers will use this to store a maximum
201 * trace while it continues examining live traces.
202 *
203 * The buffers for the max_tr are set up the same as the global_trace.
204 * When a snapshot is taken, the link list of the max_tr is swapped
205 * with the link list of the global_trace and the buffers are reset for
206 * the global_trace so the tracing can continue.
207 */
208static struct trace_array max_tr;
209
210static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
211
212/* tracer_enabled is used to toggle activation of a tracer */
213static int tracer_enabled = 1;
214
215/**
216 * tracing_is_enabled - return tracer_enabled status
217 *
218 * This function is used by other tracers to know the status
219 * of the tracer_enabled flag. Tracers may use this function
220 * to know if it should enable their features when starting
221 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
222 */
223int tracing_is_enabled(void)
224{
225 return tracer_enabled;
226}
227
228/*
229 * trace_buf_size is the size in bytes that is allocated
230 * for a buffer. Note, the number of bytes is always rounded
231 * to page size.
232 *
233 * This number is purposely set to a low number of 16384.
234 * If the dump on oops happens, it will be much appreciated
235 * to not have to wait for all that output. Anyway this can be
236 * boot time and run time configurable.
237 */
238#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
239
240static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
241
242/* trace_types holds a link list of available tracers. */
243static struct tracer *trace_types __read_mostly;
244
245/* current_trace points to the tracer that is currently active */
246static struct tracer *current_trace __read_mostly;
247
248/*
249 * trace_types_lock is used to protect the trace_types list.
250 */
251static DEFINE_MUTEX(trace_types_lock);
252
253/*
254 * serialize the access of the ring buffer
255 *
256 * ring buffer serializes readers, but it is low level protection.
257 * The validity of the events (which returns by ring_buffer_peek() ..etc)
258 * are not protected by ring buffer.
259 *
260 * The content of events may become garbage if we allow other process consumes
261 * these events concurrently:
262 * A) the page of the consumed events may become a normal page
263 * (not reader page) in ring buffer, and this page will be rewrited
264 * by events producer.
265 * B) The page of the consumed events may become a page for splice_read,
266 * and this page will be returned to system.
267 *
268 * These primitives allow multi process access to different cpu ring buffer
269 * concurrently.
270 *
271 * These primitives don't distinguish read-only and read-consume access.
272 * Multi read-only access are also serialized.
273 */
274
275#ifdef CONFIG_SMP
276static DECLARE_RWSEM(all_cpu_access_lock);
277static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
278
279static inline void trace_access_lock(int cpu)
280{
281 if (cpu == TRACE_PIPE_ALL_CPU) {
282 /* gain it for accessing the whole ring buffer. */
283 down_write(&all_cpu_access_lock);
284 } else {
285 /* gain it for accessing a cpu ring buffer. */
286
287 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
288 down_read(&all_cpu_access_lock);
289
290 /* Secondly block other access to this @cpu ring buffer. */
291 mutex_lock(&per_cpu(cpu_access_lock, cpu));
292 }
293}
294
295static inline void trace_access_unlock(int cpu)
296{
297 if (cpu == TRACE_PIPE_ALL_CPU) {
298 up_write(&all_cpu_access_lock);
299 } else {
300 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
301 up_read(&all_cpu_access_lock);
302 }
303}
304
305static inline void trace_access_lock_init(void)
306{
307 int cpu;
308
309 for_each_possible_cpu(cpu)
310 mutex_init(&per_cpu(cpu_access_lock, cpu));
311}
312
313#else
314
315static DEFINE_MUTEX(access_lock);
316
317static inline void trace_access_lock(int cpu)
318{
319 (void)cpu;
320 mutex_lock(&access_lock);
321}
322
323static inline void trace_access_unlock(int cpu)
324{
325 (void)cpu;
326 mutex_unlock(&access_lock);
327}
328
329static inline void trace_access_lock_init(void)
330{
331}
332
333#endif
334
335/* trace_wait is a waitqueue for tasks blocked on trace_poll */
336static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
337
338/* trace_flags holds trace_options default values */
339unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
340 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
341 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE;
342
343static int trace_stop_count;
344static DEFINE_SPINLOCK(tracing_start_lock);
345
346static void wakeup_work_handler(struct work_struct *work)
347{
348 wake_up(&trace_wait);
349}
350
351static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
352
353/**
354 * trace_wake_up - wake up tasks waiting for trace input
355 *
356 * Schedules a delayed work to wake up any task that is blocked on the
357 * trace_wait queue. These is used with trace_poll for tasks polling the
358 * trace.
359 */
360void trace_wake_up(void)
361{
362 const unsigned long delay = msecs_to_jiffies(2);
363
364 if (trace_flags & TRACE_ITER_BLOCK)
365 return;
366 schedule_delayed_work(&wakeup_work, delay);
367}
368
369static int __init set_buf_size(char *str)
370{
371 unsigned long buf_size;
372
373 if (!str)
374 return 0;
375 buf_size = memparse(str, &str);
376 /* nr_entries can not be zero */
377 if (buf_size == 0)
378 return 0;
379 trace_buf_size = buf_size;
380 return 1;
381}
382__setup("trace_buf_size=", set_buf_size);
383
384static int __init set_tracing_thresh(char *str)
385{
386 unsigned long threshhold;
387 int ret;
388
389 if (!str)
390 return 0;
391 ret = strict_strtoul(str, 0, &threshhold);
392 if (ret < 0)
393 return 0;
394 tracing_thresh = threshhold * 1000;
395 return 1;
396}
397__setup("tracing_thresh=", set_tracing_thresh);
398
399unsigned long nsecs_to_usecs(unsigned long nsecs)
400{
401 return nsecs / 1000;
402}
403
404/* These must match the bit postions in trace_iterator_flags */
405static const char *trace_options[] = {
406 "print-parent",
407 "sym-offset",
408 "sym-addr",
409 "verbose",
410 "raw",
411 "hex",
412 "bin",
413 "block",
414 "stacktrace",
415 "trace_printk",
416 "ftrace_preempt",
417 "branch",
418 "annotate",
419 "userstacktrace",
420 "sym-userobj",
421 "printk-msg-only",
422 "context-info",
423 "latency-format",
424 "sleep-time",
425 "graph-time",
426 "record-cmd",
427 "overwrite",
428 "disable_on_free",
429 NULL
430};
431
432static struct {
433 u64 (*func)(void);
434 const char *name;
435} trace_clocks[] = {
436 { trace_clock_local, "local" },
437 { trace_clock_global, "global" },
438};
439
440int trace_clock_id;
441
442/*
443 * trace_parser_get_init - gets the buffer for trace parser
444 */
445int trace_parser_get_init(struct trace_parser *parser, int size)
446{
447 memset(parser, 0, sizeof(*parser));
448
449 parser->buffer = kmalloc(size, GFP_KERNEL);
450 if (!parser->buffer)
451 return 1;
452
453 parser->size = size;
454 return 0;
455}
456
457/*
458 * trace_parser_put - frees the buffer for trace parser
459 */
460void trace_parser_put(struct trace_parser *parser)
461{
462 kfree(parser->buffer);
463}
464
465/*
466 * trace_get_user - reads the user input string separated by space
467 * (matched by isspace(ch))
468 *
469 * For each string found the 'struct trace_parser' is updated,
470 * and the function returns.
471 *
472 * Returns number of bytes read.
473 *
474 * See kernel/trace/trace.h for 'struct trace_parser' details.
475 */
476int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
477 size_t cnt, loff_t *ppos)
478{
479 char ch;
480 size_t read = 0;
481 ssize_t ret;
482
483 if (!*ppos)
484 trace_parser_clear(parser);
485
486 ret = get_user(ch, ubuf++);
487 if (ret)
488 goto out;
489
490 read++;
491 cnt--;
492
493 /*
494 * The parser is not finished with the last write,
495 * continue reading the user input without skipping spaces.
496 */
497 if (!parser->cont) {
498 /* skip white space */
499 while (cnt && isspace(ch)) {
500 ret = get_user(ch, ubuf++);
501 if (ret)
502 goto out;
503 read++;
504 cnt--;
505 }
506
507 /* only spaces were written */
508 if (isspace(ch)) {
509 *ppos += read;
510 ret = read;
511 goto out;
512 }
513
514 parser->idx = 0;
515 }
516
517 /* read the non-space input */
518 while (cnt && !isspace(ch)) {
519 if (parser->idx < parser->size - 1)
520 parser->buffer[parser->idx++] = ch;
521 else {
522 ret = -EINVAL;
523 goto out;
524 }
525 ret = get_user(ch, ubuf++);
526 if (ret)
527 goto out;
528 read++;
529 cnt--;
530 }
531
532 /* We either got finished input or we have to wait for another call. */
533 if (isspace(ch)) {
534 parser->buffer[parser->idx] = 0;
535 parser->cont = false;
536 } else {
537 parser->cont = true;
538 parser->buffer[parser->idx++] = ch;
539 }
540
541 *ppos += read;
542 ret = read;
543
544out:
545 return ret;
546}
547
548ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
549{
550 int len;
551 int ret;
552
553 if (!cnt)
554 return 0;
555
556 if (s->len <= s->readpos)
557 return -EBUSY;
558
559 len = s->len - s->readpos;
560 if (cnt > len)
561 cnt = len;
562 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
563 if (ret == cnt)
564 return -EFAULT;
565
566 cnt -= ret;
567
568 s->readpos += cnt;
569 return cnt;
570}
571
572static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
573{
574 int len;
575 void *ret;
576
577 if (s->len <= s->readpos)
578 return -EBUSY;
579
580 len = s->len - s->readpos;
581 if (cnt > len)
582 cnt = len;
583 ret = memcpy(buf, s->buffer + s->readpos, cnt);
584 if (!ret)
585 return -EFAULT;
586
587 s->readpos += cnt;
588 return cnt;
589}
590
591/*
592 * ftrace_max_lock is used to protect the swapping of buffers
593 * when taking a max snapshot. The buffers themselves are
594 * protected by per_cpu spinlocks. But the action of the swap
595 * needs its own lock.
596 *
597 * This is defined as a arch_spinlock_t in order to help
598 * with performance when lockdep debugging is enabled.
599 *
600 * It is also used in other places outside the update_max_tr
601 * so it needs to be defined outside of the
602 * CONFIG_TRACER_MAX_TRACE.
603 */
604static arch_spinlock_t ftrace_max_lock =
605 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
606
607unsigned long __read_mostly tracing_thresh;
608
609#ifdef CONFIG_TRACER_MAX_TRACE
610unsigned long __read_mostly tracing_max_latency;
611
612/*
613 * Copy the new maximum trace into the separate maximum-trace
614 * structure. (this way the maximum trace is permanently saved,
615 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
616 */
617static void
618__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
619{
620 struct trace_array_cpu *data = tr->data[cpu];
621 struct trace_array_cpu *max_data;
622
623 max_tr.cpu = cpu;
624 max_tr.time_start = data->preempt_timestamp;
625
626 max_data = max_tr.data[cpu];
627 max_data->saved_latency = tracing_max_latency;
628 max_data->critical_start = data->critical_start;
629 max_data->critical_end = data->critical_end;
630
631 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
632 max_data->pid = tsk->pid;
633 max_data->uid = task_uid(tsk);
634 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
635 max_data->policy = tsk->policy;
636 max_data->rt_priority = tsk->rt_priority;
637
638 /* record this tasks comm */
639 tracing_record_cmdline(tsk);
640}
641
642/**
643 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
644 * @tr: tracer
645 * @tsk: the task with the latency
646 * @cpu: The cpu that initiated the trace.
647 *
648 * Flip the buffers between the @tr and the max_tr and record information
649 * about which task was the cause of this latency.
650 */
651void
652update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
653{
654 struct ring_buffer *buf = tr->buffer;
655
656 if (trace_stop_count)
657 return;
658
659 WARN_ON_ONCE(!irqs_disabled());
660 if (!current_trace->use_max_tr) {
661 WARN_ON_ONCE(1);
662 return;
663 }
664 arch_spin_lock(&ftrace_max_lock);
665
666 tr->buffer = max_tr.buffer;
667 max_tr.buffer = buf;
668
669 __update_max_tr(tr, tsk, cpu);
670 arch_spin_unlock(&ftrace_max_lock);
671}
672
673/**
674 * update_max_tr_single - only copy one trace over, and reset the rest
675 * @tr - tracer
676 * @tsk - task with the latency
677 * @cpu - the cpu of the buffer to copy.
678 *
679 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
680 */
681void
682update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
683{
684 int ret;
685
686 if (trace_stop_count)
687 return;
688
689 WARN_ON_ONCE(!irqs_disabled());
690 if (!current_trace->use_max_tr) {
691 WARN_ON_ONCE(1);
692 return;
693 }
694
695 arch_spin_lock(&ftrace_max_lock);
696
697 ftrace_disable_cpu();
698
699 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
700
701 if (ret == -EBUSY) {
702 /*
703 * We failed to swap the buffer due to a commit taking
704 * place on this CPU. We fail to record, but we reset
705 * the max trace buffer (no one writes directly to it)
706 * and flag that it failed.
707 */
708 trace_array_printk(&max_tr, _THIS_IP_,
709 "Failed to swap buffers due to commit in progress\n");
710 }
711
712 ftrace_enable_cpu();
713
714 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
715
716 __update_max_tr(tr, tsk, cpu);
717 arch_spin_unlock(&ftrace_max_lock);
718}
719#endif /* CONFIG_TRACER_MAX_TRACE */
720
721/**
722 * register_tracer - register a tracer with the ftrace system.
723 * @type - the plugin for the tracer
724 *
725 * Register a new plugin tracer.
726 */
727int register_tracer(struct tracer *type)
728__releases(kernel_lock)
729__acquires(kernel_lock)
730{
731 struct tracer *t;
732 int ret = 0;
733
734 if (!type->name) {
735 pr_info("Tracer must have a name\n");
736 return -1;
737 }
738
739 if (strlen(type->name) >= MAX_TRACER_SIZE) {
740 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
741 return -1;
742 }
743
744 mutex_lock(&trace_types_lock);
745
746 tracing_selftest_running = true;
747
748 for (t = trace_types; t; t = t->next) {
749 if (strcmp(type->name, t->name) == 0) {
750 /* already found */
751 pr_info("Tracer %s already registered\n",
752 type->name);
753 ret = -1;
754 goto out;
755 }
756 }
757
758 if (!type->set_flag)
759 type->set_flag = &dummy_set_flag;
760 if (!type->flags)
761 type->flags = &dummy_tracer_flags;
762 else
763 if (!type->flags->opts)
764 type->flags->opts = dummy_tracer_opt;
765 if (!type->wait_pipe)
766 type->wait_pipe = default_wait_pipe;
767
768
769#ifdef CONFIG_FTRACE_STARTUP_TEST
770 if (type->selftest && !tracing_selftest_disabled) {
771 struct tracer *saved_tracer = current_trace;
772 struct trace_array *tr = &global_trace;
773
774 /*
775 * Run a selftest on this tracer.
776 * Here we reset the trace buffer, and set the current
777 * tracer to be this tracer. The tracer can then run some
778 * internal tracing to verify that everything is in order.
779 * If we fail, we do not register this tracer.
780 */
781 tracing_reset_online_cpus(tr);
782
783 current_trace = type;
784
785 /* If we expanded the buffers, make sure the max is expanded too */
786 if (ring_buffer_expanded && type->use_max_tr)
787 ring_buffer_resize(max_tr.buffer, trace_buf_size);
788
789 /* the test is responsible for initializing and enabling */
790 pr_info("Testing tracer %s: ", type->name);
791 ret = type->selftest(type, tr);
792 /* the test is responsible for resetting too */
793 current_trace = saved_tracer;
794 if (ret) {
795 printk(KERN_CONT "FAILED!\n");
796 goto out;
797 }
798 /* Only reset on passing, to avoid touching corrupted buffers */
799 tracing_reset_online_cpus(tr);
800
801 /* Shrink the max buffer again */
802 if (ring_buffer_expanded && type->use_max_tr)
803 ring_buffer_resize(max_tr.buffer, 1);
804
805 printk(KERN_CONT "PASSED\n");
806 }
807#endif
808
809 type->next = trace_types;
810 trace_types = type;
811
812 out:
813 tracing_selftest_running = false;
814 mutex_unlock(&trace_types_lock);
815
816 if (ret || !default_bootup_tracer)
817 goto out_unlock;
818
819 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
820 goto out_unlock;
821
822 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
823 /* Do we want this tracer to start on bootup? */
824 tracing_set_tracer(type->name);
825 default_bootup_tracer = NULL;
826 /* disable other selftests, since this will break it. */
827 tracing_selftest_disabled = 1;
828#ifdef CONFIG_FTRACE_STARTUP_TEST
829 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
830 type->name);
831#endif
832
833 out_unlock:
834 return ret;
835}
836
837void unregister_tracer(struct tracer *type)
838{
839 struct tracer **t;
840
841 mutex_lock(&trace_types_lock);
842 for (t = &trace_types; *t; t = &(*t)->next) {
843 if (*t == type)
844 goto found;
845 }
846 pr_info("Tracer %s not registered\n", type->name);
847 goto out;
848
849 found:
850 *t = (*t)->next;
851
852 if (type == current_trace && tracer_enabled) {
853 tracer_enabled = 0;
854 tracing_stop();
855 if (current_trace->stop)
856 current_trace->stop(&global_trace);
857 current_trace = &nop_trace;
858 }
859out:
860 mutex_unlock(&trace_types_lock);
861}
862
863static void __tracing_reset(struct ring_buffer *buffer, int cpu)
864{
865 ftrace_disable_cpu();
866 ring_buffer_reset_cpu(buffer, cpu);
867 ftrace_enable_cpu();
868}
869
870void tracing_reset(struct trace_array *tr, int cpu)
871{
872 struct ring_buffer *buffer = tr->buffer;
873
874 ring_buffer_record_disable(buffer);
875
876 /* Make sure all commits have finished */
877 synchronize_sched();
878 __tracing_reset(buffer, cpu);
879
880 ring_buffer_record_enable(buffer);
881}
882
883void tracing_reset_online_cpus(struct trace_array *tr)
884{
885 struct ring_buffer *buffer = tr->buffer;
886 int cpu;
887
888 ring_buffer_record_disable(buffer);
889
890 /* Make sure all commits have finished */
891 synchronize_sched();
892
893 tr->time_start = ftrace_now(tr->cpu);
894
895 for_each_online_cpu(cpu)
896 __tracing_reset(buffer, cpu);
897
898 ring_buffer_record_enable(buffer);
899}
900
901void tracing_reset_current(int cpu)
902{
903 tracing_reset(&global_trace, cpu);
904}
905
906void tracing_reset_current_online_cpus(void)
907{
908 tracing_reset_online_cpus(&global_trace);
909}
910
911#define SAVED_CMDLINES 128
912#define NO_CMDLINE_MAP UINT_MAX
913static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
914static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
915static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
916static int cmdline_idx;
917static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
918
919/* temporary disable recording */
920static atomic_t trace_record_cmdline_disabled __read_mostly;
921
922static void trace_init_cmdlines(void)
923{
924 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
925 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
926 cmdline_idx = 0;
927}
928
929int is_tracing_stopped(void)
930{
931 return trace_stop_count;
932}
933
934/**
935 * ftrace_off_permanent - disable all ftrace code permanently
936 *
937 * This should only be called when a serious anomally has
938 * been detected. This will turn off the function tracing,
939 * ring buffers, and other tracing utilites. It takes no
940 * locks and can be called from any context.
941 */
942void ftrace_off_permanent(void)
943{
944 tracing_disabled = 1;
945 ftrace_stop();
946 tracing_off_permanent();
947}
948
949/**
950 * tracing_start - quick start of the tracer
951 *
952 * If tracing is enabled but was stopped by tracing_stop,
953 * this will start the tracer back up.
954 */
955void tracing_start(void)
956{
957 struct ring_buffer *buffer;
958 unsigned long flags;
959
960 if (tracing_disabled)
961 return;
962
963 spin_lock_irqsave(&tracing_start_lock, flags);
964 if (--trace_stop_count) {
965 if (trace_stop_count < 0) {
966 /* Someone screwed up their debugging */
967 WARN_ON_ONCE(1);
968 trace_stop_count = 0;
969 }
970 goto out;
971 }
972
973 /* Prevent the buffers from switching */
974 arch_spin_lock(&ftrace_max_lock);
975
976 buffer = global_trace.buffer;
977 if (buffer)
978 ring_buffer_record_enable(buffer);
979
980 buffer = max_tr.buffer;
981 if (buffer)
982 ring_buffer_record_enable(buffer);
983
984 arch_spin_unlock(&ftrace_max_lock);
985
986 ftrace_start();
987 out:
988 spin_unlock_irqrestore(&tracing_start_lock, flags);
989}
990
991/**
992 * tracing_stop - quick stop of the tracer
993 *
994 * Light weight way to stop tracing. Use in conjunction with
995 * tracing_start.
996 */
997void tracing_stop(void)
998{
999 struct ring_buffer *buffer;
1000 unsigned long flags;
1001
1002 ftrace_stop();
1003 spin_lock_irqsave(&tracing_start_lock, flags);
1004 if (trace_stop_count++)
1005 goto out;
1006
1007 /* Prevent the buffers from switching */
1008 arch_spin_lock(&ftrace_max_lock);
1009
1010 buffer = global_trace.buffer;
1011 if (buffer)
1012 ring_buffer_record_disable(buffer);
1013
1014 buffer = max_tr.buffer;
1015 if (buffer)
1016 ring_buffer_record_disable(buffer);
1017
1018 arch_spin_unlock(&ftrace_max_lock);
1019
1020 out:
1021 spin_unlock_irqrestore(&tracing_start_lock, flags);
1022}
1023
1024void trace_stop_cmdline_recording(void);
1025
1026static void trace_save_cmdline(struct task_struct *tsk)
1027{
1028 unsigned pid, idx;
1029
1030 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1031 return;
1032
1033 /*
1034 * It's not the end of the world if we don't get
1035 * the lock, but we also don't want to spin
1036 * nor do we want to disable interrupts,
1037 * so if we miss here, then better luck next time.
1038 */
1039 if (!arch_spin_trylock(&trace_cmdline_lock))
1040 return;
1041
1042 idx = map_pid_to_cmdline[tsk->pid];
1043 if (idx == NO_CMDLINE_MAP) {
1044 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1045
1046 /*
1047 * Check whether the cmdline buffer at idx has a pid
1048 * mapped. We are going to overwrite that entry so we
1049 * need to clear the map_pid_to_cmdline. Otherwise we
1050 * would read the new comm for the old pid.
1051 */
1052 pid = map_cmdline_to_pid[idx];
1053 if (pid != NO_CMDLINE_MAP)
1054 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1055
1056 map_cmdline_to_pid[idx] = tsk->pid;
1057 map_pid_to_cmdline[tsk->pid] = idx;
1058
1059 cmdline_idx = idx;
1060 }
1061
1062 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1063
1064 arch_spin_unlock(&trace_cmdline_lock);
1065}
1066
1067void trace_find_cmdline(int pid, char comm[])
1068{
1069 unsigned map;
1070
1071 if (!pid) {
1072 strcpy(comm, "<idle>");
1073 return;
1074 }
1075
1076 if (WARN_ON_ONCE(pid < 0)) {
1077 strcpy(comm, "<XXX>");
1078 return;
1079 }
1080
1081 if (pid > PID_MAX_DEFAULT) {
1082 strcpy(comm, "<...>");
1083 return;
1084 }
1085
1086 preempt_disable();
1087 arch_spin_lock(&trace_cmdline_lock);
1088 map = map_pid_to_cmdline[pid];
1089 if (map != NO_CMDLINE_MAP)
1090 strcpy(comm, saved_cmdlines[map]);
1091 else
1092 strcpy(comm, "<...>");
1093
1094 arch_spin_unlock(&trace_cmdline_lock);
1095 preempt_enable();
1096}
1097
1098void tracing_record_cmdline(struct task_struct *tsk)
1099{
1100 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1101 !tracing_is_on())
1102 return;
1103
1104 trace_save_cmdline(tsk);
1105}
1106
1107void
1108tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1109 int pc)
1110{
1111 struct task_struct *tsk = current;
1112
1113 entry->preempt_count = pc & 0xff;
1114 entry->pid = (tsk) ? tsk->pid : 0;
1115 entry->padding = 0;
1116 entry->flags =
1117#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1118 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1119#else
1120 TRACE_FLAG_IRQS_NOSUPPORT |
1121#endif
1122 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1123 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1124 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1125}
1126EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1127
1128struct ring_buffer_event *
1129trace_buffer_lock_reserve(struct ring_buffer *buffer,
1130 int type,
1131 unsigned long len,
1132 unsigned long flags, int pc)
1133{
1134 struct ring_buffer_event *event;
1135
1136 event = ring_buffer_lock_reserve(buffer, len);
1137 if (event != NULL) {
1138 struct trace_entry *ent = ring_buffer_event_data(event);
1139
1140 tracing_generic_entry_update(ent, flags, pc);
1141 ent->type = type;
1142 }
1143
1144 return event;
1145}
1146
1147static inline void
1148__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1149 struct ring_buffer_event *event,
1150 unsigned long flags, int pc,
1151 int wake)
1152{
1153 ring_buffer_unlock_commit(buffer, event);
1154
1155 ftrace_trace_stack(buffer, flags, 6, pc);
1156 ftrace_trace_userstack(buffer, flags, pc);
1157
1158 if (wake)
1159 trace_wake_up();
1160}
1161
1162void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1163 struct ring_buffer_event *event,
1164 unsigned long flags, int pc)
1165{
1166 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1167}
1168
1169struct ring_buffer_event *
1170trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1171 int type, unsigned long len,
1172 unsigned long flags, int pc)
1173{
1174 *current_rb = global_trace.buffer;
1175 return trace_buffer_lock_reserve(*current_rb,
1176 type, len, flags, pc);
1177}
1178EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1179
1180void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1181 struct ring_buffer_event *event,
1182 unsigned long flags, int pc)
1183{
1184 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1185}
1186EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1187
1188void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1189 struct ring_buffer_event *event,
1190 unsigned long flags, int pc)
1191{
1192 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1193}
1194EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1195
1196void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1197 struct ring_buffer_event *event,
1198 unsigned long flags, int pc,
1199 struct pt_regs *regs)
1200{
1201 ring_buffer_unlock_commit(buffer, event);
1202
1203 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1204 ftrace_trace_userstack(buffer, flags, pc);
1205}
1206EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1207
1208void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1209 struct ring_buffer_event *event)
1210{
1211 ring_buffer_discard_commit(buffer, event);
1212}
1213EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1214
1215void
1216trace_function(struct trace_array *tr,
1217 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1218 int pc)
1219{
1220 struct ftrace_event_call *call = &event_function;
1221 struct ring_buffer *buffer = tr->buffer;
1222 struct ring_buffer_event *event;
1223 struct ftrace_entry *entry;
1224
1225 /* If we are reading the ring buffer, don't trace */
1226 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1227 return;
1228
1229 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1230 flags, pc);
1231 if (!event)
1232 return;
1233 entry = ring_buffer_event_data(event);
1234 entry->ip = ip;
1235 entry->parent_ip = parent_ip;
1236
1237 if (!filter_check_discard(call, entry, buffer, event))
1238 ring_buffer_unlock_commit(buffer, event);
1239}
1240
1241void
1242ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1243 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1244 int pc)
1245{
1246 if (likely(!atomic_read(&data->disabled)))
1247 trace_function(tr, ip, parent_ip, flags, pc);
1248}
1249
1250#ifdef CONFIG_STACKTRACE
1251
1252#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1253struct ftrace_stack {
1254 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1255};
1256
1257static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1258static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1259
1260static void __ftrace_trace_stack(struct ring_buffer *buffer,
1261 unsigned long flags,
1262 int skip, int pc, struct pt_regs *regs)
1263{
1264 struct ftrace_event_call *call = &event_kernel_stack;
1265 struct ring_buffer_event *event;
1266 struct stack_entry *entry;
1267 struct stack_trace trace;
1268 int use_stack;
1269 int size = FTRACE_STACK_ENTRIES;
1270
1271 trace.nr_entries = 0;
1272 trace.skip = skip;
1273
1274 /*
1275 * Since events can happen in NMIs there's no safe way to
1276 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1277 * or NMI comes in, it will just have to use the default
1278 * FTRACE_STACK_SIZE.
1279 */
1280 preempt_disable_notrace();
1281
1282 use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1283 /*
1284 * We don't need any atomic variables, just a barrier.
1285 * If an interrupt comes in, we don't care, because it would
1286 * have exited and put the counter back to what we want.
1287 * We just need a barrier to keep gcc from moving things
1288 * around.
1289 */
1290 barrier();
1291 if (use_stack == 1) {
1292 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1293 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1294
1295 if (regs)
1296 save_stack_trace_regs(regs, &trace);
1297 else
1298 save_stack_trace(&trace);
1299
1300 if (trace.nr_entries > size)
1301 size = trace.nr_entries;
1302 } else
1303 /* From now on, use_stack is a boolean */
1304 use_stack = 0;
1305
1306 size *= sizeof(unsigned long);
1307
1308 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1309 sizeof(*entry) + size, flags, pc);
1310 if (!event)
1311 goto out;
1312 entry = ring_buffer_event_data(event);
1313
1314 memset(&entry->caller, 0, size);
1315
1316 if (use_stack)
1317 memcpy(&entry->caller, trace.entries,
1318 trace.nr_entries * sizeof(unsigned long));
1319 else {
1320 trace.max_entries = FTRACE_STACK_ENTRIES;
1321 trace.entries = entry->caller;
1322 if (regs)
1323 save_stack_trace_regs(regs, &trace);
1324 else
1325 save_stack_trace(&trace);
1326 }
1327
1328 entry->size = trace.nr_entries;
1329
1330 if (!filter_check_discard(call, entry, buffer, event))
1331 ring_buffer_unlock_commit(buffer, event);
1332
1333 out:
1334 /* Again, don't let gcc optimize things here */
1335 barrier();
1336 __get_cpu_var(ftrace_stack_reserve)--;
1337 preempt_enable_notrace();
1338
1339}
1340
1341void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1342 int skip, int pc, struct pt_regs *regs)
1343{
1344 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1345 return;
1346
1347 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1348}
1349
1350void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1351 int skip, int pc)
1352{
1353 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1354 return;
1355
1356 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1357}
1358
1359void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1360 int pc)
1361{
1362 __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1363}
1364
1365/**
1366 * trace_dump_stack - record a stack back trace in the trace buffer
1367 */
1368void trace_dump_stack(void)
1369{
1370 unsigned long flags;
1371
1372 if (tracing_disabled || tracing_selftest_running)
1373 return;
1374
1375 local_save_flags(flags);
1376
1377 /* skipping 3 traces, seems to get us at the caller of this function */
1378 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1379}
1380
1381static DEFINE_PER_CPU(int, user_stack_count);
1382
1383void
1384ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1385{
1386 struct ftrace_event_call *call = &event_user_stack;
1387 struct ring_buffer_event *event;
1388 struct userstack_entry *entry;
1389 struct stack_trace trace;
1390
1391 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1392 return;
1393
1394 /*
1395 * NMIs can not handle page faults, even with fix ups.
1396 * The save user stack can (and often does) fault.
1397 */
1398 if (unlikely(in_nmi()))
1399 return;
1400
1401 /*
1402 * prevent recursion, since the user stack tracing may
1403 * trigger other kernel events.
1404 */
1405 preempt_disable();
1406 if (__this_cpu_read(user_stack_count))
1407 goto out;
1408
1409 __this_cpu_inc(user_stack_count);
1410
1411 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1412 sizeof(*entry), flags, pc);
1413 if (!event)
1414 goto out_drop_count;
1415 entry = ring_buffer_event_data(event);
1416
1417 entry->tgid = current->tgid;
1418 memset(&entry->caller, 0, sizeof(entry->caller));
1419
1420 trace.nr_entries = 0;
1421 trace.max_entries = FTRACE_STACK_ENTRIES;
1422 trace.skip = 0;
1423 trace.entries = entry->caller;
1424
1425 save_stack_trace_user(&trace);
1426 if (!filter_check_discard(call, entry, buffer, event))
1427 ring_buffer_unlock_commit(buffer, event);
1428
1429 out_drop_count:
1430 __this_cpu_dec(user_stack_count);
1431 out:
1432 preempt_enable();
1433}
1434
1435#ifdef UNUSED
1436static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1437{
1438 ftrace_trace_userstack(tr, flags, preempt_count());
1439}
1440#endif /* UNUSED */
1441
1442#endif /* CONFIG_STACKTRACE */
1443
1444/**
1445 * trace_vbprintk - write binary msg to tracing buffer
1446 *
1447 */
1448int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1449{
1450 static arch_spinlock_t trace_buf_lock =
1451 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1452 static u32 trace_buf[TRACE_BUF_SIZE];
1453
1454 struct ftrace_event_call *call = &event_bprint;
1455 struct ring_buffer_event *event;
1456 struct ring_buffer *buffer;
1457 struct trace_array *tr = &global_trace;
1458 struct trace_array_cpu *data;
1459 struct bprint_entry *entry;
1460 unsigned long flags;
1461 int disable;
1462 int cpu, len = 0, size, pc;
1463
1464 if (unlikely(tracing_selftest_running || tracing_disabled))
1465 return 0;
1466
1467 /* Don't pollute graph traces with trace_vprintk internals */
1468 pause_graph_tracing();
1469
1470 pc = preempt_count();
1471 preempt_disable_notrace();
1472 cpu = raw_smp_processor_id();
1473 data = tr->data[cpu];
1474
1475 disable = atomic_inc_return(&data->disabled);
1476 if (unlikely(disable != 1))
1477 goto out;
1478
1479 /* Lockdep uses trace_printk for lock tracing */
1480 local_irq_save(flags);
1481 arch_spin_lock(&trace_buf_lock);
1482 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1483
1484 if (len > TRACE_BUF_SIZE || len < 0)
1485 goto out_unlock;
1486
1487 size = sizeof(*entry) + sizeof(u32) * len;
1488 buffer = tr->buffer;
1489 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1490 flags, pc);
1491 if (!event)
1492 goto out_unlock;
1493 entry = ring_buffer_event_data(event);
1494 entry->ip = ip;
1495 entry->fmt = fmt;
1496
1497 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1498 if (!filter_check_discard(call, entry, buffer, event)) {
1499 ring_buffer_unlock_commit(buffer, event);
1500 ftrace_trace_stack(buffer, flags, 6, pc);
1501 }
1502
1503out_unlock:
1504 arch_spin_unlock(&trace_buf_lock);
1505 local_irq_restore(flags);
1506
1507out:
1508 atomic_dec_return(&data->disabled);
1509 preempt_enable_notrace();
1510 unpause_graph_tracing();
1511
1512 return len;
1513}
1514EXPORT_SYMBOL_GPL(trace_vbprintk);
1515
1516int trace_array_printk(struct trace_array *tr,
1517 unsigned long ip, const char *fmt, ...)
1518{
1519 int ret;
1520 va_list ap;
1521
1522 if (!(trace_flags & TRACE_ITER_PRINTK))
1523 return 0;
1524
1525 va_start(ap, fmt);
1526 ret = trace_array_vprintk(tr, ip, fmt, ap);
1527 va_end(ap);
1528 return ret;
1529}
1530
1531int trace_array_vprintk(struct trace_array *tr,
1532 unsigned long ip, const char *fmt, va_list args)
1533{
1534 static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1535 static char trace_buf[TRACE_BUF_SIZE];
1536
1537 struct ftrace_event_call *call = &event_print;
1538 struct ring_buffer_event *event;
1539 struct ring_buffer *buffer;
1540 struct trace_array_cpu *data;
1541 int cpu, len = 0, size, pc;
1542 struct print_entry *entry;
1543 unsigned long irq_flags;
1544 int disable;
1545
1546 if (tracing_disabled || tracing_selftest_running)
1547 return 0;
1548
1549 pc = preempt_count();
1550 preempt_disable_notrace();
1551 cpu = raw_smp_processor_id();
1552 data = tr->data[cpu];
1553
1554 disable = atomic_inc_return(&data->disabled);
1555 if (unlikely(disable != 1))
1556 goto out;
1557
1558 pause_graph_tracing();
1559 raw_local_irq_save(irq_flags);
1560 arch_spin_lock(&trace_buf_lock);
1561 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1562
1563 size = sizeof(*entry) + len + 1;
1564 buffer = tr->buffer;
1565 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1566 irq_flags, pc);
1567 if (!event)
1568 goto out_unlock;
1569 entry = ring_buffer_event_data(event);
1570 entry->ip = ip;
1571
1572 memcpy(&entry->buf, trace_buf, len);
1573 entry->buf[len] = '\0';
1574 if (!filter_check_discard(call, entry, buffer, event)) {
1575 ring_buffer_unlock_commit(buffer, event);
1576 ftrace_trace_stack(buffer, irq_flags, 6, pc);
1577 }
1578
1579 out_unlock:
1580 arch_spin_unlock(&trace_buf_lock);
1581 raw_local_irq_restore(irq_flags);
1582 unpause_graph_tracing();
1583 out:
1584 atomic_dec_return(&data->disabled);
1585 preempt_enable_notrace();
1586
1587 return len;
1588}
1589
1590int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1591{
1592 return trace_array_vprintk(&global_trace, ip, fmt, args);
1593}
1594EXPORT_SYMBOL_GPL(trace_vprintk);
1595
1596static void trace_iterator_increment(struct trace_iterator *iter)
1597{
1598 /* Don't allow ftrace to trace into the ring buffers */
1599 ftrace_disable_cpu();
1600
1601 iter->idx++;
1602 if (iter->buffer_iter[iter->cpu])
1603 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1604
1605 ftrace_enable_cpu();
1606}
1607
1608static struct trace_entry *
1609peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1610 unsigned long *lost_events)
1611{
1612 struct ring_buffer_event *event;
1613 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1614
1615 /* Don't allow ftrace to trace into the ring buffers */
1616 ftrace_disable_cpu();
1617
1618 if (buf_iter)
1619 event = ring_buffer_iter_peek(buf_iter, ts);
1620 else
1621 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1622 lost_events);
1623
1624 ftrace_enable_cpu();
1625
1626 if (event) {
1627 iter->ent_size = ring_buffer_event_length(event);
1628 return ring_buffer_event_data(event);
1629 }
1630 iter->ent_size = 0;
1631 return NULL;
1632}
1633
1634static struct trace_entry *
1635__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1636 unsigned long *missing_events, u64 *ent_ts)
1637{
1638 struct ring_buffer *buffer = iter->tr->buffer;
1639 struct trace_entry *ent, *next = NULL;
1640 unsigned long lost_events = 0, next_lost = 0;
1641 int cpu_file = iter->cpu_file;
1642 u64 next_ts = 0, ts;
1643 int next_cpu = -1;
1644 int cpu;
1645
1646 /*
1647 * If we are in a per_cpu trace file, don't bother by iterating over
1648 * all cpu and peek directly.
1649 */
1650 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1651 if (ring_buffer_empty_cpu(buffer, cpu_file))
1652 return NULL;
1653 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1654 if (ent_cpu)
1655 *ent_cpu = cpu_file;
1656
1657 return ent;
1658 }
1659
1660 for_each_tracing_cpu(cpu) {
1661
1662 if (ring_buffer_empty_cpu(buffer, cpu))
1663 continue;
1664
1665 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1666
1667 /*
1668 * Pick the entry with the smallest timestamp:
1669 */
1670 if (ent && (!next || ts < next_ts)) {
1671 next = ent;
1672 next_cpu = cpu;
1673 next_ts = ts;
1674 next_lost = lost_events;
1675 }
1676 }
1677
1678 if (ent_cpu)
1679 *ent_cpu = next_cpu;
1680
1681 if (ent_ts)
1682 *ent_ts = next_ts;
1683
1684 if (missing_events)
1685 *missing_events = next_lost;
1686
1687 return next;
1688}
1689
1690/* Find the next real entry, without updating the iterator itself */
1691struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1692 int *ent_cpu, u64 *ent_ts)
1693{
1694 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1695}
1696
1697/* Find the next real entry, and increment the iterator to the next entry */
1698void *trace_find_next_entry_inc(struct trace_iterator *iter)
1699{
1700 iter->ent = __find_next_entry(iter, &iter->cpu,
1701 &iter->lost_events, &iter->ts);
1702
1703 if (iter->ent)
1704 trace_iterator_increment(iter);
1705
1706 return iter->ent ? iter : NULL;
1707}
1708
1709static void trace_consume(struct trace_iterator *iter)
1710{
1711 /* Don't allow ftrace to trace into the ring buffers */
1712 ftrace_disable_cpu();
1713 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1714 &iter->lost_events);
1715 ftrace_enable_cpu();
1716}
1717
1718static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1719{
1720 struct trace_iterator *iter = m->private;
1721 int i = (int)*pos;
1722 void *ent;
1723
1724 WARN_ON_ONCE(iter->leftover);
1725
1726 (*pos)++;
1727
1728 /* can't go backwards */
1729 if (iter->idx > i)
1730 return NULL;
1731
1732 if (iter->idx < 0)
1733 ent = trace_find_next_entry_inc(iter);
1734 else
1735 ent = iter;
1736
1737 while (ent && iter->idx < i)
1738 ent = trace_find_next_entry_inc(iter);
1739
1740 iter->pos = *pos;
1741
1742 return ent;
1743}
1744
1745void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1746{
1747 struct trace_array *tr = iter->tr;
1748 struct ring_buffer_event *event;
1749 struct ring_buffer_iter *buf_iter;
1750 unsigned long entries = 0;
1751 u64 ts;
1752
1753 tr->data[cpu]->skipped_entries = 0;
1754
1755 if (!iter->buffer_iter[cpu])
1756 return;
1757
1758 buf_iter = iter->buffer_iter[cpu];
1759 ring_buffer_iter_reset(buf_iter);
1760
1761 /*
1762 * We could have the case with the max latency tracers
1763 * that a reset never took place on a cpu. This is evident
1764 * by the timestamp being before the start of the buffer.
1765 */
1766 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1767 if (ts >= iter->tr->time_start)
1768 break;
1769 entries++;
1770 ring_buffer_read(buf_iter, NULL);
1771 }
1772
1773 tr->data[cpu]->skipped_entries = entries;
1774}
1775
1776/*
1777 * The current tracer is copied to avoid a global locking
1778 * all around.
1779 */
1780static void *s_start(struct seq_file *m, loff_t *pos)
1781{
1782 struct trace_iterator *iter = m->private;
1783 static struct tracer *old_tracer;
1784 int cpu_file = iter->cpu_file;
1785 void *p = NULL;
1786 loff_t l = 0;
1787 int cpu;
1788
1789 /* copy the tracer to avoid using a global lock all around */
1790 mutex_lock(&trace_types_lock);
1791 if (unlikely(old_tracer != current_trace && current_trace)) {
1792 old_tracer = current_trace;
1793 *iter->trace = *current_trace;
1794 }
1795 mutex_unlock(&trace_types_lock);
1796
1797 atomic_inc(&trace_record_cmdline_disabled);
1798
1799 if (*pos != iter->pos) {
1800 iter->ent = NULL;
1801 iter->cpu = 0;
1802 iter->idx = -1;
1803
1804 ftrace_disable_cpu();
1805
1806 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1807 for_each_tracing_cpu(cpu)
1808 tracing_iter_reset(iter, cpu);
1809 } else
1810 tracing_iter_reset(iter, cpu_file);
1811
1812 ftrace_enable_cpu();
1813
1814 iter->leftover = 0;
1815 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1816 ;
1817
1818 } else {
1819 /*
1820 * If we overflowed the seq_file before, then we want
1821 * to just reuse the trace_seq buffer again.
1822 */
1823 if (iter->leftover)
1824 p = iter;
1825 else {
1826 l = *pos - 1;
1827 p = s_next(m, p, &l);
1828 }
1829 }
1830
1831 trace_event_read_lock();
1832 trace_access_lock(cpu_file);
1833 return p;
1834}
1835
1836static void s_stop(struct seq_file *m, void *p)
1837{
1838 struct trace_iterator *iter = m->private;
1839
1840 atomic_dec(&trace_record_cmdline_disabled);
1841 trace_access_unlock(iter->cpu_file);
1842 trace_event_read_unlock();
1843}
1844
1845static void print_lat_help_header(struct seq_file *m)
1846{
1847 seq_puts(m, "# _------=> CPU# \n");
1848 seq_puts(m, "# / _-----=> irqs-off \n");
1849 seq_puts(m, "# | / _----=> need-resched \n");
1850 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1851 seq_puts(m, "# ||| / _--=> preempt-depth \n");
1852 seq_puts(m, "# |||| / delay \n");
1853 seq_puts(m, "# cmd pid ||||| time | caller \n");
1854 seq_puts(m, "# \\ / ||||| \\ | / \n");
1855}
1856
1857static void print_func_help_header(struct seq_file *m)
1858{
1859 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1860 seq_puts(m, "# | | | | |\n");
1861}
1862
1863
1864void
1865print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1866{
1867 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1868 struct trace_array *tr = iter->tr;
1869 struct trace_array_cpu *data = tr->data[tr->cpu];
1870 struct tracer *type = current_trace;
1871 unsigned long entries = 0;
1872 unsigned long total = 0;
1873 unsigned long count;
1874 const char *name = "preemption";
1875 int cpu;
1876
1877 if (type)
1878 name = type->name;
1879
1880
1881 for_each_tracing_cpu(cpu) {
1882 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1883 /*
1884 * If this buffer has skipped entries, then we hold all
1885 * entries for the trace and we need to ignore the
1886 * ones before the time stamp.
1887 */
1888 if (tr->data[cpu]->skipped_entries) {
1889 count -= tr->data[cpu]->skipped_entries;
1890 /* total is the same as the entries */
1891 total += count;
1892 } else
1893 total += count +
1894 ring_buffer_overrun_cpu(tr->buffer, cpu);
1895 entries += count;
1896 }
1897
1898 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1899 name, UTS_RELEASE);
1900 seq_puts(m, "# -----------------------------------"
1901 "---------------------------------\n");
1902 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
1903 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1904 nsecs_to_usecs(data->saved_latency),
1905 entries,
1906 total,
1907 tr->cpu,
1908#if defined(CONFIG_PREEMPT_NONE)
1909 "server",
1910#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1911 "desktop",
1912#elif defined(CONFIG_PREEMPT)
1913 "preempt",
1914#else
1915 "unknown",
1916#endif
1917 /* These are reserved for later use */
1918 0, 0, 0, 0);
1919#ifdef CONFIG_SMP
1920 seq_printf(m, " #P:%d)\n", num_online_cpus());
1921#else
1922 seq_puts(m, ")\n");
1923#endif
1924 seq_puts(m, "# -----------------\n");
1925 seq_printf(m, "# | task: %.16s-%d "
1926 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1927 data->comm, data->pid, data->uid, data->nice,
1928 data->policy, data->rt_priority);
1929 seq_puts(m, "# -----------------\n");
1930
1931 if (data->critical_start) {
1932 seq_puts(m, "# => started at: ");
1933 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1934 trace_print_seq(m, &iter->seq);
1935 seq_puts(m, "\n# => ended at: ");
1936 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1937 trace_print_seq(m, &iter->seq);
1938 seq_puts(m, "\n#\n");
1939 }
1940
1941 seq_puts(m, "#\n");
1942}
1943
1944static void test_cpu_buff_start(struct trace_iterator *iter)
1945{
1946 struct trace_seq *s = &iter->seq;
1947
1948 if (!(trace_flags & TRACE_ITER_ANNOTATE))
1949 return;
1950
1951 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1952 return;
1953
1954 if (cpumask_test_cpu(iter->cpu, iter->started))
1955 return;
1956
1957 if (iter->tr->data[iter->cpu]->skipped_entries)
1958 return;
1959
1960 cpumask_set_cpu(iter->cpu, iter->started);
1961
1962 /* Don't print started cpu buffer for the first entry of the trace */
1963 if (iter->idx > 1)
1964 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1965 iter->cpu);
1966}
1967
1968static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1969{
1970 struct trace_seq *s = &iter->seq;
1971 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1972 struct trace_entry *entry;
1973 struct trace_event *event;
1974
1975 entry = iter->ent;
1976
1977 test_cpu_buff_start(iter);
1978
1979 event = ftrace_find_event(entry->type);
1980
1981 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1982 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1983 if (!trace_print_lat_context(iter))
1984 goto partial;
1985 } else {
1986 if (!trace_print_context(iter))
1987 goto partial;
1988 }
1989 }
1990
1991 if (event)
1992 return event->funcs->trace(iter, sym_flags, event);
1993
1994 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1995 goto partial;
1996
1997 return TRACE_TYPE_HANDLED;
1998partial:
1999 return TRACE_TYPE_PARTIAL_LINE;
2000}
2001
2002static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2003{
2004 struct trace_seq *s = &iter->seq;
2005 struct trace_entry *entry;
2006 struct trace_event *event;
2007
2008 entry = iter->ent;
2009
2010 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2011 if (!trace_seq_printf(s, "%d %d %llu ",
2012 entry->pid, iter->cpu, iter->ts))
2013 goto partial;
2014 }
2015
2016 event = ftrace_find_event(entry->type);
2017 if (event)
2018 return event->funcs->raw(iter, 0, event);
2019
2020 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2021 goto partial;
2022
2023 return TRACE_TYPE_HANDLED;
2024partial:
2025 return TRACE_TYPE_PARTIAL_LINE;
2026}
2027
2028static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2029{
2030 struct trace_seq *s = &iter->seq;
2031 unsigned char newline = '\n';
2032 struct trace_entry *entry;
2033 struct trace_event *event;
2034
2035 entry = iter->ent;
2036
2037 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2038 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2039 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2040 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2041 }
2042
2043 event = ftrace_find_event(entry->type);
2044 if (event) {
2045 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2046 if (ret != TRACE_TYPE_HANDLED)
2047 return ret;
2048 }
2049
2050 SEQ_PUT_FIELD_RET(s, newline);
2051
2052 return TRACE_TYPE_HANDLED;
2053}
2054
2055static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2056{
2057 struct trace_seq *s = &iter->seq;
2058 struct trace_entry *entry;
2059 struct trace_event *event;
2060
2061 entry = iter->ent;
2062
2063 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2064 SEQ_PUT_FIELD_RET(s, entry->pid);
2065 SEQ_PUT_FIELD_RET(s, iter->cpu);
2066 SEQ_PUT_FIELD_RET(s, iter->ts);
2067 }
2068
2069 event = ftrace_find_event(entry->type);
2070 return event ? event->funcs->binary(iter, 0, event) :
2071 TRACE_TYPE_HANDLED;
2072}
2073
2074int trace_empty(struct trace_iterator *iter)
2075{
2076 int cpu;
2077
2078 /* If we are looking at one CPU buffer, only check that one */
2079 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2080 cpu = iter->cpu_file;
2081 if (iter->buffer_iter[cpu]) {
2082 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2083 return 0;
2084 } else {
2085 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2086 return 0;
2087 }
2088 return 1;
2089 }
2090
2091 for_each_tracing_cpu(cpu) {
2092 if (iter->buffer_iter[cpu]) {
2093 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2094 return 0;
2095 } else {
2096 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2097 return 0;
2098 }
2099 }
2100
2101 return 1;
2102}
2103
2104/* Called with trace_event_read_lock() held. */
2105enum print_line_t print_trace_line(struct trace_iterator *iter)
2106{
2107 enum print_line_t ret;
2108
2109 if (iter->lost_events &&
2110 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2111 iter->cpu, iter->lost_events))
2112 return TRACE_TYPE_PARTIAL_LINE;
2113
2114 if (iter->trace && iter->trace->print_line) {
2115 ret = iter->trace->print_line(iter);
2116 if (ret != TRACE_TYPE_UNHANDLED)
2117 return ret;
2118 }
2119
2120 if (iter->ent->type == TRACE_BPRINT &&
2121 trace_flags & TRACE_ITER_PRINTK &&
2122 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2123 return trace_print_bprintk_msg_only(iter);
2124
2125 if (iter->ent->type == TRACE_PRINT &&
2126 trace_flags & TRACE_ITER_PRINTK &&
2127 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2128 return trace_print_printk_msg_only(iter);
2129
2130 if (trace_flags & TRACE_ITER_BIN)
2131 return print_bin_fmt(iter);
2132
2133 if (trace_flags & TRACE_ITER_HEX)
2134 return print_hex_fmt(iter);
2135
2136 if (trace_flags & TRACE_ITER_RAW)
2137 return print_raw_fmt(iter);
2138
2139 return print_trace_fmt(iter);
2140}
2141
2142void trace_default_header(struct seq_file *m)
2143{
2144 struct trace_iterator *iter = m->private;
2145
2146 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2147 return;
2148
2149 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2150 /* print nothing if the buffers are empty */
2151 if (trace_empty(iter))
2152 return;
2153 print_trace_header(m, iter);
2154 if (!(trace_flags & TRACE_ITER_VERBOSE))
2155 print_lat_help_header(m);
2156 } else {
2157 if (!(trace_flags & TRACE_ITER_VERBOSE))
2158 print_func_help_header(m);
2159 }
2160}
2161
2162static int s_show(struct seq_file *m, void *v)
2163{
2164 struct trace_iterator *iter = v;
2165 int ret;
2166
2167 if (iter->ent == NULL) {
2168 if (iter->tr) {
2169 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2170 seq_puts(m, "#\n");
2171 }
2172 if (iter->trace && iter->trace->print_header)
2173 iter->trace->print_header(m);
2174 else
2175 trace_default_header(m);
2176
2177 } else if (iter->leftover) {
2178 /*
2179 * If we filled the seq_file buffer earlier, we
2180 * want to just show it now.
2181 */
2182 ret = trace_print_seq(m, &iter->seq);
2183
2184 /* ret should this time be zero, but you never know */
2185 iter->leftover = ret;
2186
2187 } else {
2188 print_trace_line(iter);
2189 ret = trace_print_seq(m, &iter->seq);
2190 /*
2191 * If we overflow the seq_file buffer, then it will
2192 * ask us for this data again at start up.
2193 * Use that instead.
2194 * ret is 0 if seq_file write succeeded.
2195 * -1 otherwise.
2196 */
2197 iter->leftover = ret;
2198 }
2199
2200 return 0;
2201}
2202
2203static const struct seq_operations tracer_seq_ops = {
2204 .start = s_start,
2205 .next = s_next,
2206 .stop = s_stop,
2207 .show = s_show,
2208};
2209
2210static struct trace_iterator *
2211__tracing_open(struct inode *inode, struct file *file)
2212{
2213 long cpu_file = (long) inode->i_private;
2214 void *fail_ret = ERR_PTR(-ENOMEM);
2215 struct trace_iterator *iter;
2216 struct seq_file *m;
2217 int cpu, ret;
2218
2219 if (tracing_disabled)
2220 return ERR_PTR(-ENODEV);
2221
2222 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2223 if (!iter)
2224 return ERR_PTR(-ENOMEM);
2225
2226 /*
2227 * We make a copy of the current tracer to avoid concurrent
2228 * changes on it while we are reading.
2229 */
2230 mutex_lock(&trace_types_lock);
2231 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2232 if (!iter->trace)
2233 goto fail;
2234
2235 if (current_trace)
2236 *iter->trace = *current_trace;
2237
2238 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2239 goto fail;
2240
2241 if (current_trace && current_trace->print_max)
2242 iter->tr = &max_tr;
2243 else
2244 iter->tr = &global_trace;
2245 iter->pos = -1;
2246 mutex_init(&iter->mutex);
2247 iter->cpu_file = cpu_file;
2248
2249 /* Notify the tracer early; before we stop tracing. */
2250 if (iter->trace && iter->trace->open)
2251 iter->trace->open(iter);
2252
2253 /* Annotate start of buffers if we had overruns */
2254 if (ring_buffer_overruns(iter->tr->buffer))
2255 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2256
2257 /* stop the trace while dumping */
2258 tracing_stop();
2259
2260 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2261 for_each_tracing_cpu(cpu) {
2262 iter->buffer_iter[cpu] =
2263 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2264 }
2265 ring_buffer_read_prepare_sync();
2266 for_each_tracing_cpu(cpu) {
2267 ring_buffer_read_start(iter->buffer_iter[cpu]);
2268 tracing_iter_reset(iter, cpu);
2269 }
2270 } else {
2271 cpu = iter->cpu_file;
2272 iter->buffer_iter[cpu] =
2273 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2274 ring_buffer_read_prepare_sync();
2275 ring_buffer_read_start(iter->buffer_iter[cpu]);
2276 tracing_iter_reset(iter, cpu);
2277 }
2278
2279 ret = seq_open(file, &tracer_seq_ops);
2280 if (ret < 0) {
2281 fail_ret = ERR_PTR(ret);
2282 goto fail_buffer;
2283 }
2284
2285 m = file->private_data;
2286 m->private = iter;
2287
2288 mutex_unlock(&trace_types_lock);
2289
2290 return iter;
2291
2292 fail_buffer:
2293 for_each_tracing_cpu(cpu) {
2294 if (iter->buffer_iter[cpu])
2295 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2296 }
2297 free_cpumask_var(iter->started);
2298 tracing_start();
2299 fail:
2300 mutex_unlock(&trace_types_lock);
2301 kfree(iter->trace);
2302 kfree(iter);
2303
2304 return fail_ret;
2305}
2306
2307int tracing_open_generic(struct inode *inode, struct file *filp)
2308{
2309 if (tracing_disabled)
2310 return -ENODEV;
2311
2312 filp->private_data = inode->i_private;
2313 return 0;
2314}
2315
2316static int tracing_release(struct inode *inode, struct file *file)
2317{
2318 struct seq_file *m = file->private_data;
2319 struct trace_iterator *iter;
2320 int cpu;
2321
2322 if (!(file->f_mode & FMODE_READ))
2323 return 0;
2324
2325 iter = m->private;
2326
2327 mutex_lock(&trace_types_lock);
2328 for_each_tracing_cpu(cpu) {
2329 if (iter->buffer_iter[cpu])
2330 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2331 }
2332
2333 if (iter->trace && iter->trace->close)
2334 iter->trace->close(iter);
2335
2336 /* reenable tracing if it was previously enabled */
2337 tracing_start();
2338 mutex_unlock(&trace_types_lock);
2339
2340 seq_release(inode, file);
2341 mutex_destroy(&iter->mutex);
2342 free_cpumask_var(iter->started);
2343 kfree(iter->trace);
2344 kfree(iter);
2345 return 0;
2346}
2347
2348static int tracing_open(struct inode *inode, struct file *file)
2349{
2350 struct trace_iterator *iter;
2351 int ret = 0;
2352
2353 /* If this file was open for write, then erase contents */
2354 if ((file->f_mode & FMODE_WRITE) &&
2355 (file->f_flags & O_TRUNC)) {
2356 long cpu = (long) inode->i_private;
2357
2358 if (cpu == TRACE_PIPE_ALL_CPU)
2359 tracing_reset_online_cpus(&global_trace);
2360 else
2361 tracing_reset(&global_trace, cpu);
2362 }
2363
2364 if (file->f_mode & FMODE_READ) {
2365 iter = __tracing_open(inode, file);
2366 if (IS_ERR(iter))
2367 ret = PTR_ERR(iter);
2368 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2369 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2370 }
2371 return ret;
2372}
2373
2374static void *
2375t_next(struct seq_file *m, void *v, loff_t *pos)
2376{
2377 struct tracer *t = v;
2378
2379 (*pos)++;
2380
2381 if (t)
2382 t = t->next;
2383
2384 return t;
2385}
2386
2387static void *t_start(struct seq_file *m, loff_t *pos)
2388{
2389 struct tracer *t;
2390 loff_t l = 0;
2391
2392 mutex_lock(&trace_types_lock);
2393 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2394 ;
2395
2396 return t;
2397}
2398
2399static void t_stop(struct seq_file *m, void *p)
2400{
2401 mutex_unlock(&trace_types_lock);
2402}
2403
2404static int t_show(struct seq_file *m, void *v)
2405{
2406 struct tracer *t = v;
2407
2408 if (!t)
2409 return 0;
2410
2411 seq_printf(m, "%s", t->name);
2412 if (t->next)
2413 seq_putc(m, ' ');
2414 else
2415 seq_putc(m, '\n');
2416
2417 return 0;
2418}
2419
2420static const struct seq_operations show_traces_seq_ops = {
2421 .start = t_start,
2422 .next = t_next,
2423 .stop = t_stop,
2424 .show = t_show,
2425};
2426
2427static int show_traces_open(struct inode *inode, struct file *file)
2428{
2429 if (tracing_disabled)
2430 return -ENODEV;
2431
2432 return seq_open(file, &show_traces_seq_ops);
2433}
2434
2435static ssize_t
2436tracing_write_stub(struct file *filp, const char __user *ubuf,
2437 size_t count, loff_t *ppos)
2438{
2439 return count;
2440}
2441
2442static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2443{
2444 if (file->f_mode & FMODE_READ)
2445 return seq_lseek(file, offset, origin);
2446 else
2447 return 0;
2448}
2449
2450static const struct file_operations tracing_fops = {
2451 .open = tracing_open,
2452 .read = seq_read,
2453 .write = tracing_write_stub,
2454 .llseek = tracing_seek,
2455 .release = tracing_release,
2456};
2457
2458static const struct file_operations show_traces_fops = {
2459 .open = show_traces_open,
2460 .read = seq_read,
2461 .release = seq_release,
2462 .llseek = seq_lseek,
2463};
2464
2465/*
2466 * Only trace on a CPU if the bitmask is set:
2467 */
2468static cpumask_var_t tracing_cpumask;
2469
2470/*
2471 * The tracer itself will not take this lock, but still we want
2472 * to provide a consistent cpumask to user-space:
2473 */
2474static DEFINE_MUTEX(tracing_cpumask_update_lock);
2475
2476/*
2477 * Temporary storage for the character representation of the
2478 * CPU bitmask (and one more byte for the newline):
2479 */
2480static char mask_str[NR_CPUS + 1];
2481
2482static ssize_t
2483tracing_cpumask_read(struct file *filp, char __user *ubuf,
2484 size_t count, loff_t *ppos)
2485{
2486 int len;
2487
2488 mutex_lock(&tracing_cpumask_update_lock);
2489
2490 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2491 if (count - len < 2) {
2492 count = -EINVAL;
2493 goto out_err;
2494 }
2495 len += sprintf(mask_str + len, "\n");
2496 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2497
2498out_err:
2499 mutex_unlock(&tracing_cpumask_update_lock);
2500
2501 return count;
2502}
2503
2504static ssize_t
2505tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2506 size_t count, loff_t *ppos)
2507{
2508 int err, cpu;
2509 cpumask_var_t tracing_cpumask_new;
2510
2511 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2512 return -ENOMEM;
2513
2514 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2515 if (err)
2516 goto err_unlock;
2517
2518 mutex_lock(&tracing_cpumask_update_lock);
2519
2520 local_irq_disable();
2521 arch_spin_lock(&ftrace_max_lock);
2522 for_each_tracing_cpu(cpu) {
2523 /*
2524 * Increase/decrease the disabled counter if we are
2525 * about to flip a bit in the cpumask:
2526 */
2527 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2528 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2529 atomic_inc(&global_trace.data[cpu]->disabled);
2530 }
2531 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2532 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2533 atomic_dec(&global_trace.data[cpu]->disabled);
2534 }
2535 }
2536 arch_spin_unlock(&ftrace_max_lock);
2537 local_irq_enable();
2538
2539 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2540
2541 mutex_unlock(&tracing_cpumask_update_lock);
2542 free_cpumask_var(tracing_cpumask_new);
2543
2544 return count;
2545
2546err_unlock:
2547 free_cpumask_var(tracing_cpumask_new);
2548
2549 return err;
2550}
2551
2552static const struct file_operations tracing_cpumask_fops = {
2553 .open = tracing_open_generic,
2554 .read = tracing_cpumask_read,
2555 .write = tracing_cpumask_write,
2556 .llseek = generic_file_llseek,
2557};
2558
2559static int tracing_trace_options_show(struct seq_file *m, void *v)
2560{
2561 struct tracer_opt *trace_opts;
2562 u32 tracer_flags;
2563 int i;
2564
2565 mutex_lock(&trace_types_lock);
2566 tracer_flags = current_trace->flags->val;
2567 trace_opts = current_trace->flags->opts;
2568
2569 for (i = 0; trace_options[i]; i++) {
2570 if (trace_flags & (1 << i))
2571 seq_printf(m, "%s\n", trace_options[i]);
2572 else
2573 seq_printf(m, "no%s\n", trace_options[i]);
2574 }
2575
2576 for (i = 0; trace_opts[i].name; i++) {
2577 if (tracer_flags & trace_opts[i].bit)
2578 seq_printf(m, "%s\n", trace_opts[i].name);
2579 else
2580 seq_printf(m, "no%s\n", trace_opts[i].name);
2581 }
2582 mutex_unlock(&trace_types_lock);
2583
2584 return 0;
2585}
2586
2587static int __set_tracer_option(struct tracer *trace,
2588 struct tracer_flags *tracer_flags,
2589 struct tracer_opt *opts, int neg)
2590{
2591 int ret;
2592
2593 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2594 if (ret)
2595 return ret;
2596
2597 if (neg)
2598 tracer_flags->val &= ~opts->bit;
2599 else
2600 tracer_flags->val |= opts->bit;
2601 return 0;
2602}
2603
2604/* Try to assign a tracer specific option */
2605static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2606{
2607 struct tracer_flags *tracer_flags = trace->flags;
2608 struct tracer_opt *opts = NULL;
2609 int i;
2610
2611 for (i = 0; tracer_flags->opts[i].name; i++) {
2612 opts = &tracer_flags->opts[i];
2613
2614 if (strcmp(cmp, opts->name) == 0)
2615 return __set_tracer_option(trace, trace->flags,
2616 opts, neg);
2617 }
2618
2619 return -EINVAL;
2620}
2621
2622static void set_tracer_flags(unsigned int mask, int enabled)
2623{
2624 /* do nothing if flag is already set */
2625 if (!!(trace_flags & mask) == !!enabled)
2626 return;
2627
2628 if (enabled)
2629 trace_flags |= mask;
2630 else
2631 trace_flags &= ~mask;
2632
2633 if (mask == TRACE_ITER_RECORD_CMD)
2634 trace_event_enable_cmd_record(enabled);
2635
2636 if (mask == TRACE_ITER_OVERWRITE)
2637 ring_buffer_change_overwrite(global_trace.buffer, enabled);
2638}
2639
2640static ssize_t
2641tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2642 size_t cnt, loff_t *ppos)
2643{
2644 char buf[64];
2645 char *cmp;
2646 int neg = 0;
2647 int ret;
2648 int i;
2649
2650 if (cnt >= sizeof(buf))
2651 return -EINVAL;
2652
2653 if (copy_from_user(&buf, ubuf, cnt))
2654 return -EFAULT;
2655
2656 buf[cnt] = 0;
2657 cmp = strstrip(buf);
2658
2659 if (strncmp(cmp, "no", 2) == 0) {
2660 neg = 1;
2661 cmp += 2;
2662 }
2663
2664 for (i = 0; trace_options[i]; i++) {
2665 if (strcmp(cmp, trace_options[i]) == 0) {
2666 set_tracer_flags(1 << i, !neg);
2667 break;
2668 }
2669 }
2670
2671 /* If no option could be set, test the specific tracer options */
2672 if (!trace_options[i]) {
2673 mutex_lock(&trace_types_lock);
2674 ret = set_tracer_option(current_trace, cmp, neg);
2675 mutex_unlock(&trace_types_lock);
2676 if (ret)
2677 return ret;
2678 }
2679
2680 *ppos += cnt;
2681
2682 return cnt;
2683}
2684
2685static int tracing_trace_options_open(struct inode *inode, struct file *file)
2686{
2687 if (tracing_disabled)
2688 return -ENODEV;
2689 return single_open(file, tracing_trace_options_show, NULL);
2690}
2691
2692static const struct file_operations tracing_iter_fops = {
2693 .open = tracing_trace_options_open,
2694 .read = seq_read,
2695 .llseek = seq_lseek,
2696 .release = single_release,
2697 .write = tracing_trace_options_write,
2698};
2699
2700static const char readme_msg[] =
2701 "tracing mini-HOWTO:\n\n"
2702 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2703 "# cat /sys/kernel/debug/tracing/available_tracers\n"
2704 "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2705 "# cat /sys/kernel/debug/tracing/current_tracer\n"
2706 "nop\n"
2707 "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2708 "# cat /sys/kernel/debug/tracing/current_tracer\n"
2709 "sched_switch\n"
2710 "# cat /sys/kernel/debug/tracing/trace_options\n"
2711 "noprint-parent nosym-offset nosym-addr noverbose\n"
2712 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2713 "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2714 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2715 "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
2716;
2717
2718static ssize_t
2719tracing_readme_read(struct file *filp, char __user *ubuf,
2720 size_t cnt, loff_t *ppos)
2721{
2722 return simple_read_from_buffer(ubuf, cnt, ppos,
2723 readme_msg, strlen(readme_msg));
2724}
2725
2726static const struct file_operations tracing_readme_fops = {
2727 .open = tracing_open_generic,
2728 .read = tracing_readme_read,
2729 .llseek = generic_file_llseek,
2730};
2731
2732static ssize_t
2733tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2734 size_t cnt, loff_t *ppos)
2735{
2736 char *buf_comm;
2737 char *file_buf;
2738 char *buf;
2739 int len = 0;
2740 int pid;
2741 int i;
2742
2743 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2744 if (!file_buf)
2745 return -ENOMEM;
2746
2747 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2748 if (!buf_comm) {
2749 kfree(file_buf);
2750 return -ENOMEM;
2751 }
2752
2753 buf = file_buf;
2754
2755 for (i = 0; i < SAVED_CMDLINES; i++) {
2756 int r;
2757
2758 pid = map_cmdline_to_pid[i];
2759 if (pid == -1 || pid == NO_CMDLINE_MAP)
2760 continue;
2761
2762 trace_find_cmdline(pid, buf_comm);
2763 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2764 buf += r;
2765 len += r;
2766 }
2767
2768 len = simple_read_from_buffer(ubuf, cnt, ppos,
2769 file_buf, len);
2770
2771 kfree(file_buf);
2772 kfree(buf_comm);
2773
2774 return len;
2775}
2776
2777static const struct file_operations tracing_saved_cmdlines_fops = {
2778 .open = tracing_open_generic,
2779 .read = tracing_saved_cmdlines_read,
2780 .llseek = generic_file_llseek,
2781};
2782
2783static ssize_t
2784tracing_ctrl_read(struct file *filp, char __user *ubuf,
2785 size_t cnt, loff_t *ppos)
2786{
2787 char buf[64];
2788 int r;
2789
2790 r = sprintf(buf, "%u\n", tracer_enabled);
2791 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2792}
2793
2794static ssize_t
2795tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2796 size_t cnt, loff_t *ppos)
2797{
2798 struct trace_array *tr = filp->private_data;
2799 unsigned long val;
2800 int ret;
2801
2802 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2803 if (ret)
2804 return ret;
2805
2806 val = !!val;
2807
2808 mutex_lock(&trace_types_lock);
2809 if (tracer_enabled ^ val) {
2810
2811 /* Only need to warn if this is used to change the state */
2812 WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2813
2814 if (val) {
2815 tracer_enabled = 1;
2816 if (current_trace->start)
2817 current_trace->start(tr);
2818 tracing_start();
2819 } else {
2820 tracer_enabled = 0;
2821 tracing_stop();
2822 if (current_trace->stop)
2823 current_trace->stop(tr);
2824 }
2825 }
2826 mutex_unlock(&trace_types_lock);
2827
2828 *ppos += cnt;
2829
2830 return cnt;
2831}
2832
2833static ssize_t
2834tracing_set_trace_read(struct file *filp, char __user *ubuf,
2835 size_t cnt, loff_t *ppos)
2836{
2837 char buf[MAX_TRACER_SIZE+2];
2838 int r;
2839
2840 mutex_lock(&trace_types_lock);
2841 if (current_trace)
2842 r = sprintf(buf, "%s\n", current_trace->name);
2843 else
2844 r = sprintf(buf, "\n");
2845 mutex_unlock(&trace_types_lock);
2846
2847 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2848}
2849
2850int tracer_init(struct tracer *t, struct trace_array *tr)
2851{
2852 tracing_reset_online_cpus(tr);
2853 return t->init(tr);
2854}
2855
2856static int __tracing_resize_ring_buffer(unsigned long size)
2857{
2858 int ret;
2859
2860 /*
2861 * If kernel or user changes the size of the ring buffer
2862 * we use the size that was given, and we can forget about
2863 * expanding it later.
2864 */
2865 ring_buffer_expanded = 1;
2866
2867 ret = ring_buffer_resize(global_trace.buffer, size);
2868 if (ret < 0)
2869 return ret;
2870
2871 if (!current_trace->use_max_tr)
2872 goto out;
2873
2874 ret = ring_buffer_resize(max_tr.buffer, size);
2875 if (ret < 0) {
2876 int r;
2877
2878 r = ring_buffer_resize(global_trace.buffer,
2879 global_trace.entries);
2880 if (r < 0) {
2881 /*
2882 * AARGH! We are left with different
2883 * size max buffer!!!!
2884 * The max buffer is our "snapshot" buffer.
2885 * When a tracer needs a snapshot (one of the
2886 * latency tracers), it swaps the max buffer
2887 * with the saved snap shot. We succeeded to
2888 * update the size of the main buffer, but failed to
2889 * update the size of the max buffer. But when we tried
2890 * to reset the main buffer to the original size, we
2891 * failed there too. This is very unlikely to
2892 * happen, but if it does, warn and kill all
2893 * tracing.
2894 */
2895 WARN_ON(1);
2896 tracing_disabled = 1;
2897 }
2898 return ret;
2899 }
2900
2901 max_tr.entries = size;
2902 out:
2903 global_trace.entries = size;
2904
2905 return ret;
2906}
2907
2908static ssize_t tracing_resize_ring_buffer(unsigned long size)
2909{
2910 int cpu, ret = size;
2911
2912 mutex_lock(&trace_types_lock);
2913
2914 tracing_stop();
2915
2916 /* disable all cpu buffers */
2917 for_each_tracing_cpu(cpu) {
2918 if (global_trace.data[cpu])
2919 atomic_inc(&global_trace.data[cpu]->disabled);
2920 if (max_tr.data[cpu])
2921 atomic_inc(&max_tr.data[cpu]->disabled);
2922 }
2923
2924 if (size != global_trace.entries)
2925 ret = __tracing_resize_ring_buffer(size);
2926
2927 if (ret < 0)
2928 ret = -ENOMEM;
2929
2930 for_each_tracing_cpu(cpu) {
2931 if (global_trace.data[cpu])
2932 atomic_dec(&global_trace.data[cpu]->disabled);
2933 if (max_tr.data[cpu])
2934 atomic_dec(&max_tr.data[cpu]->disabled);
2935 }
2936
2937 tracing_start();
2938 mutex_unlock(&trace_types_lock);
2939
2940 return ret;
2941}
2942
2943
2944/**
2945 * tracing_update_buffers - used by tracing facility to expand ring buffers
2946 *
2947 * To save on memory when the tracing is never used on a system with it
2948 * configured in. The ring buffers are set to a minimum size. But once
2949 * a user starts to use the tracing facility, then they need to grow
2950 * to their default size.
2951 *
2952 * This function is to be called when a tracer is about to be used.
2953 */
2954int tracing_update_buffers(void)
2955{
2956 int ret = 0;
2957
2958 mutex_lock(&trace_types_lock);
2959 if (!ring_buffer_expanded)
2960 ret = __tracing_resize_ring_buffer(trace_buf_size);
2961 mutex_unlock(&trace_types_lock);
2962
2963 return ret;
2964}
2965
2966struct trace_option_dentry;
2967
2968static struct trace_option_dentry *
2969create_trace_option_files(struct tracer *tracer);
2970
2971static void
2972destroy_trace_option_files(struct trace_option_dentry *topts);
2973
2974static int tracing_set_tracer(const char *buf)
2975{
2976 static struct trace_option_dentry *topts;
2977 struct trace_array *tr = &global_trace;
2978 struct tracer *t;
2979 int ret = 0;
2980
2981 mutex_lock(&trace_types_lock);
2982
2983 if (!ring_buffer_expanded) {
2984 ret = __tracing_resize_ring_buffer(trace_buf_size);
2985 if (ret < 0)
2986 goto out;
2987 ret = 0;
2988 }
2989
2990 for (t = trace_types; t; t = t->next) {
2991 if (strcmp(t->name, buf) == 0)
2992 break;
2993 }
2994 if (!t) {
2995 ret = -EINVAL;
2996 goto out;
2997 }
2998 if (t == current_trace)
2999 goto out;
3000
3001 trace_branch_disable();
3002 if (current_trace && current_trace->reset)
3003 current_trace->reset(tr);
3004 if (current_trace && current_trace->use_max_tr) {
3005 /*
3006 * We don't free the ring buffer. instead, resize it because
3007 * The max_tr ring buffer has some state (e.g. ring->clock) and
3008 * we want preserve it.
3009 */
3010 ring_buffer_resize(max_tr.buffer, 1);
3011 max_tr.entries = 1;
3012 }
3013 destroy_trace_option_files(topts);
3014
3015 current_trace = t;
3016
3017 topts = create_trace_option_files(current_trace);
3018 if (current_trace->use_max_tr) {
3019 ret = ring_buffer_resize(max_tr.buffer, global_trace.entries);
3020 if (ret < 0)
3021 goto out;
3022 max_tr.entries = global_trace.entries;
3023 }
3024
3025 if (t->init) {
3026 ret = tracer_init(t, tr);
3027 if (ret)
3028 goto out;
3029 }
3030
3031 trace_branch_enable(tr);
3032 out:
3033 mutex_unlock(&trace_types_lock);
3034
3035 return ret;
3036}
3037
3038static ssize_t
3039tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3040 size_t cnt, loff_t *ppos)
3041{
3042 char buf[MAX_TRACER_SIZE+1];
3043 int i;
3044 size_t ret;
3045 int err;
3046
3047 ret = cnt;
3048
3049 if (cnt > MAX_TRACER_SIZE)
3050 cnt = MAX_TRACER_SIZE;
3051
3052 if (copy_from_user(&buf, ubuf, cnt))
3053 return -EFAULT;
3054
3055 buf[cnt] = 0;
3056
3057 /* strip ending whitespace. */
3058 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3059 buf[i] = 0;
3060
3061 err = tracing_set_tracer(buf);
3062 if (err)
3063 return err;
3064
3065 *ppos += ret;
3066
3067 return ret;
3068}
3069
3070static ssize_t
3071tracing_max_lat_read(struct file *filp, char __user *ubuf,
3072 size_t cnt, loff_t *ppos)
3073{
3074 unsigned long *ptr = filp->private_data;
3075 char buf[64];
3076 int r;
3077
3078 r = snprintf(buf, sizeof(buf), "%ld\n",
3079 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3080 if (r > sizeof(buf))
3081 r = sizeof(buf);
3082 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3083}
3084
3085static ssize_t
3086tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3087 size_t cnt, loff_t *ppos)
3088{
3089 unsigned long *ptr = filp->private_data;
3090 unsigned long val;
3091 int ret;
3092
3093 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3094 if (ret)
3095 return ret;
3096
3097 *ptr = val * 1000;
3098
3099 return cnt;
3100}
3101
3102static int tracing_open_pipe(struct inode *inode, struct file *filp)
3103{
3104 long cpu_file = (long) inode->i_private;
3105 struct trace_iterator *iter;
3106 int ret = 0;
3107
3108 if (tracing_disabled)
3109 return -ENODEV;
3110
3111 mutex_lock(&trace_types_lock);
3112
3113 /* create a buffer to store the information to pass to userspace */
3114 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3115 if (!iter) {
3116 ret = -ENOMEM;
3117 goto out;
3118 }
3119
3120 /*
3121 * We make a copy of the current tracer to avoid concurrent
3122 * changes on it while we are reading.
3123 */
3124 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3125 if (!iter->trace) {
3126 ret = -ENOMEM;
3127 goto fail;
3128 }
3129 if (current_trace)
3130 *iter->trace = *current_trace;
3131
3132 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3133 ret = -ENOMEM;
3134 goto fail;
3135 }
3136
3137 /* trace pipe does not show start of buffer */
3138 cpumask_setall(iter->started);
3139
3140 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3141 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3142
3143 iter->cpu_file = cpu_file;
3144 iter->tr = &global_trace;
3145 mutex_init(&iter->mutex);
3146 filp->private_data = iter;
3147
3148 if (iter->trace->pipe_open)
3149 iter->trace->pipe_open(iter);
3150
3151 nonseekable_open(inode, filp);
3152out:
3153 mutex_unlock(&trace_types_lock);
3154 return ret;
3155
3156fail:
3157 kfree(iter->trace);
3158 kfree(iter);
3159 mutex_unlock(&trace_types_lock);
3160 return ret;
3161}
3162
3163static int tracing_release_pipe(struct inode *inode, struct file *file)
3164{
3165 struct trace_iterator *iter = file->private_data;
3166
3167 mutex_lock(&trace_types_lock);
3168
3169 if (iter->trace->pipe_close)
3170 iter->trace->pipe_close(iter);
3171
3172 mutex_unlock(&trace_types_lock);
3173
3174 free_cpumask_var(iter->started);
3175 mutex_destroy(&iter->mutex);
3176 kfree(iter->trace);
3177 kfree(iter);
3178
3179 return 0;
3180}
3181
3182static unsigned int
3183tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3184{
3185 struct trace_iterator *iter = filp->private_data;
3186
3187 if (trace_flags & TRACE_ITER_BLOCK) {
3188 /*
3189 * Always select as readable when in blocking mode
3190 */
3191 return POLLIN | POLLRDNORM;
3192 } else {
3193 if (!trace_empty(iter))
3194 return POLLIN | POLLRDNORM;
3195 poll_wait(filp, &trace_wait, poll_table);
3196 if (!trace_empty(iter))
3197 return POLLIN | POLLRDNORM;
3198
3199 return 0;
3200 }
3201}
3202
3203
3204void default_wait_pipe(struct trace_iterator *iter)
3205{
3206 DEFINE_WAIT(wait);
3207
3208 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3209
3210 if (trace_empty(iter))
3211 schedule();
3212
3213 finish_wait(&trace_wait, &wait);
3214}
3215
3216/*
3217 * This is a make-shift waitqueue.
3218 * A tracer might use this callback on some rare cases:
3219 *
3220 * 1) the current tracer might hold the runqueue lock when it wakes up
3221 * a reader, hence a deadlock (sched, function, and function graph tracers)
3222 * 2) the function tracers, trace all functions, we don't want
3223 * the overhead of calling wake_up and friends
3224 * (and tracing them too)
3225 *
3226 * Anyway, this is really very primitive wakeup.
3227 */
3228void poll_wait_pipe(struct trace_iterator *iter)
3229{
3230 set_current_state(TASK_INTERRUPTIBLE);
3231 /* sleep for 100 msecs, and try again. */
3232 schedule_timeout(HZ / 10);
3233}
3234
3235/* Must be called with trace_types_lock mutex held. */
3236static int tracing_wait_pipe(struct file *filp)
3237{
3238 struct trace_iterator *iter = filp->private_data;
3239
3240 while (trace_empty(iter)) {
3241
3242 if ((filp->f_flags & O_NONBLOCK)) {
3243 return -EAGAIN;
3244 }
3245
3246 mutex_unlock(&iter->mutex);
3247
3248 iter->trace->wait_pipe(iter);
3249
3250 mutex_lock(&iter->mutex);
3251
3252 if (signal_pending(current))
3253 return -EINTR;
3254
3255 /*
3256 * We block until we read something and tracing is disabled.
3257 * We still block if tracing is disabled, but we have never
3258 * read anything. This allows a user to cat this file, and
3259 * then enable tracing. But after we have read something,
3260 * we give an EOF when tracing is again disabled.
3261 *
3262 * iter->pos will be 0 if we haven't read anything.
3263 */
3264 if (!tracer_enabled && iter->pos)
3265 break;
3266 }
3267
3268 return 1;
3269}
3270
3271/*
3272 * Consumer reader.
3273 */
3274static ssize_t
3275tracing_read_pipe(struct file *filp, char __user *ubuf,
3276 size_t cnt, loff_t *ppos)
3277{
3278 struct trace_iterator *iter = filp->private_data;
3279 static struct tracer *old_tracer;
3280 ssize_t sret;
3281
3282 /* return any leftover data */
3283 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3284 if (sret != -EBUSY)
3285 return sret;
3286
3287 trace_seq_init(&iter->seq);
3288
3289 /* copy the tracer to avoid using a global lock all around */
3290 mutex_lock(&trace_types_lock);
3291 if (unlikely(old_tracer != current_trace && current_trace)) {
3292 old_tracer = current_trace;
3293 *iter->trace = *current_trace;
3294 }
3295 mutex_unlock(&trace_types_lock);
3296
3297 /*
3298 * Avoid more than one consumer on a single file descriptor
3299 * This is just a matter of traces coherency, the ring buffer itself
3300 * is protected.
3301 */
3302 mutex_lock(&iter->mutex);
3303 if (iter->trace->read) {
3304 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3305 if (sret)
3306 goto out;
3307 }
3308
3309waitagain:
3310 sret = tracing_wait_pipe(filp);
3311 if (sret <= 0)
3312 goto out;
3313
3314 /* stop when tracing is finished */
3315 if (trace_empty(iter)) {
3316 sret = 0;
3317 goto out;
3318 }
3319
3320 if (cnt >= PAGE_SIZE)
3321 cnt = PAGE_SIZE - 1;
3322
3323 /* reset all but tr, trace, and overruns */
3324 memset(&iter->seq, 0,
3325 sizeof(struct trace_iterator) -
3326 offsetof(struct trace_iterator, seq));
3327 iter->pos = -1;
3328
3329 trace_event_read_lock();
3330 trace_access_lock(iter->cpu_file);
3331 while (trace_find_next_entry_inc(iter) != NULL) {
3332 enum print_line_t ret;
3333 int len = iter->seq.len;
3334
3335 ret = print_trace_line(iter);
3336 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3337 /* don't print partial lines */
3338 iter->seq.len = len;
3339 break;
3340 }
3341 if (ret != TRACE_TYPE_NO_CONSUME)
3342 trace_consume(iter);
3343
3344 if (iter->seq.len >= cnt)
3345 break;
3346
3347 /*
3348 * Setting the full flag means we reached the trace_seq buffer
3349 * size and we should leave by partial output condition above.
3350 * One of the trace_seq_* functions is not used properly.
3351 */
3352 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3353 iter->ent->type);
3354 }
3355 trace_access_unlock(iter->cpu_file);
3356 trace_event_read_unlock();
3357
3358 /* Now copy what we have to the user */
3359 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3360 if (iter->seq.readpos >= iter->seq.len)
3361 trace_seq_init(&iter->seq);
3362
3363 /*
3364 * If there was nothing to send to user, in spite of consuming trace
3365 * entries, go back to wait for more entries.
3366 */
3367 if (sret == -EBUSY)
3368 goto waitagain;
3369
3370out:
3371 mutex_unlock(&iter->mutex);
3372
3373 return sret;
3374}
3375
3376static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3377 struct pipe_buffer *buf)
3378{
3379 __free_page(buf->page);
3380}
3381
3382static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3383 unsigned int idx)
3384{
3385 __free_page(spd->pages[idx]);
3386}
3387
3388static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3389 .can_merge = 0,
3390 .map = generic_pipe_buf_map,
3391 .unmap = generic_pipe_buf_unmap,
3392 .confirm = generic_pipe_buf_confirm,
3393 .release = tracing_pipe_buf_release,
3394 .steal = generic_pipe_buf_steal,
3395 .get = generic_pipe_buf_get,
3396};
3397
3398static size_t
3399tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3400{
3401 size_t count;
3402 int ret;
3403
3404 /* Seq buffer is page-sized, exactly what we need. */
3405 for (;;) {
3406 count = iter->seq.len;
3407 ret = print_trace_line(iter);
3408 count = iter->seq.len - count;
3409 if (rem < count) {
3410 rem = 0;
3411 iter->seq.len -= count;
3412 break;
3413 }
3414 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3415 iter->seq.len -= count;
3416 break;
3417 }
3418
3419 if (ret != TRACE_TYPE_NO_CONSUME)
3420 trace_consume(iter);
3421 rem -= count;
3422 if (!trace_find_next_entry_inc(iter)) {
3423 rem = 0;
3424 iter->ent = NULL;
3425 break;
3426 }
3427 }
3428
3429 return rem;
3430}
3431
3432static ssize_t tracing_splice_read_pipe(struct file *filp,
3433 loff_t *ppos,
3434 struct pipe_inode_info *pipe,
3435 size_t len,
3436 unsigned int flags)
3437{
3438 struct page *pages_def[PIPE_DEF_BUFFERS];
3439 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3440 struct trace_iterator *iter = filp->private_data;
3441 struct splice_pipe_desc spd = {
3442 .pages = pages_def,
3443 .partial = partial_def,
3444 .nr_pages = 0, /* This gets updated below. */
3445 .flags = flags,
3446 .ops = &tracing_pipe_buf_ops,
3447 .spd_release = tracing_spd_release_pipe,
3448 };
3449 static struct tracer *old_tracer;
3450 ssize_t ret;
3451 size_t rem;
3452 unsigned int i;
3453
3454 if (splice_grow_spd(pipe, &spd))
3455 return -ENOMEM;
3456
3457 /* copy the tracer to avoid using a global lock all around */
3458 mutex_lock(&trace_types_lock);
3459 if (unlikely(old_tracer != current_trace && current_trace)) {
3460 old_tracer = current_trace;
3461 *iter->trace = *current_trace;
3462 }
3463 mutex_unlock(&trace_types_lock);
3464
3465 mutex_lock(&iter->mutex);
3466
3467 if (iter->trace->splice_read) {
3468 ret = iter->trace->splice_read(iter, filp,
3469 ppos, pipe, len, flags);
3470 if (ret)
3471 goto out_err;
3472 }
3473
3474 ret = tracing_wait_pipe(filp);
3475 if (ret <= 0)
3476 goto out_err;
3477
3478 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3479 ret = -EFAULT;
3480 goto out_err;
3481 }
3482
3483 trace_event_read_lock();
3484 trace_access_lock(iter->cpu_file);
3485
3486 /* Fill as many pages as possible. */
3487 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3488 spd.pages[i] = alloc_page(GFP_KERNEL);
3489 if (!spd.pages[i])
3490 break;
3491
3492 rem = tracing_fill_pipe_page(rem, iter);
3493
3494 /* Copy the data into the page, so we can start over. */
3495 ret = trace_seq_to_buffer(&iter->seq,
3496 page_address(spd.pages[i]),
3497 iter->seq.len);
3498 if (ret < 0) {
3499 __free_page(spd.pages[i]);
3500 break;
3501 }
3502 spd.partial[i].offset = 0;
3503 spd.partial[i].len = iter->seq.len;
3504
3505 trace_seq_init(&iter->seq);
3506 }
3507
3508 trace_access_unlock(iter->cpu_file);
3509 trace_event_read_unlock();
3510 mutex_unlock(&iter->mutex);
3511
3512 spd.nr_pages = i;
3513
3514 ret = splice_to_pipe(pipe, &spd);
3515out:
3516 splice_shrink_spd(pipe, &spd);
3517 return ret;
3518
3519out_err:
3520 mutex_unlock(&iter->mutex);
3521 goto out;
3522}
3523
3524static ssize_t
3525tracing_entries_read(struct file *filp, char __user *ubuf,
3526 size_t cnt, loff_t *ppos)
3527{
3528 struct trace_array *tr = filp->private_data;
3529 char buf[96];
3530 int r;
3531
3532 mutex_lock(&trace_types_lock);
3533 if (!ring_buffer_expanded)
3534 r = sprintf(buf, "%lu (expanded: %lu)\n",
3535 tr->entries >> 10,
3536 trace_buf_size >> 10);
3537 else
3538 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3539 mutex_unlock(&trace_types_lock);
3540
3541 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3542}
3543
3544static ssize_t
3545tracing_entries_write(struct file *filp, const char __user *ubuf,
3546 size_t cnt, loff_t *ppos)
3547{
3548 unsigned long val;
3549 int ret;
3550
3551 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3552 if (ret)
3553 return ret;
3554
3555 /* must have at least 1 entry */
3556 if (!val)
3557 return -EINVAL;
3558
3559 /* value is in KB */
3560 val <<= 10;
3561
3562 ret = tracing_resize_ring_buffer(val);
3563 if (ret < 0)
3564 return ret;
3565
3566 *ppos += cnt;
3567
3568 return cnt;
3569}
3570
3571static ssize_t
3572tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3573 size_t cnt, loff_t *ppos)
3574{
3575 /*
3576 * There is no need to read what the user has written, this function
3577 * is just to make sure that there is no error when "echo" is used
3578 */
3579
3580 *ppos += cnt;
3581
3582 return cnt;
3583}
3584
3585static int
3586tracing_free_buffer_release(struct inode *inode, struct file *filp)
3587{
3588 /* disable tracing ? */
3589 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3590 tracing_off();
3591 /* resize the ring buffer to 0 */
3592 tracing_resize_ring_buffer(0);
3593
3594 return 0;
3595}
3596
3597static int mark_printk(const char *fmt, ...)
3598{
3599 int ret;
3600 va_list args;
3601 va_start(args, fmt);
3602 ret = trace_vprintk(0, fmt, args);
3603 va_end(args);
3604 return ret;
3605}
3606
3607static ssize_t
3608tracing_mark_write(struct file *filp, const char __user *ubuf,
3609 size_t cnt, loff_t *fpos)
3610{
3611 char *buf;
3612 size_t written;
3613
3614 if (tracing_disabled)
3615 return -EINVAL;
3616
3617 if (cnt > TRACE_BUF_SIZE)
3618 cnt = TRACE_BUF_SIZE;
3619
3620 buf = kmalloc(cnt + 2, GFP_KERNEL);
3621 if (buf == NULL)
3622 return -ENOMEM;
3623
3624 if (copy_from_user(buf, ubuf, cnt)) {
3625 kfree(buf);
3626 return -EFAULT;
3627 }
3628 if (buf[cnt-1] != '\n') {
3629 buf[cnt] = '\n';
3630 buf[cnt+1] = '\0';
3631 } else
3632 buf[cnt] = '\0';
3633
3634 written = mark_printk("%s", buf);
3635 kfree(buf);
3636 *fpos += written;
3637
3638 /* don't tell userspace we wrote more - it might confuse them */
3639 if (written > cnt)
3640 written = cnt;
3641
3642 return written;
3643}
3644
3645static int tracing_clock_show(struct seq_file *m, void *v)
3646{
3647 int i;
3648
3649 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3650 seq_printf(m,
3651 "%s%s%s%s", i ? " " : "",
3652 i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3653 i == trace_clock_id ? "]" : "");
3654 seq_putc(m, '\n');
3655
3656 return 0;
3657}
3658
3659static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3660 size_t cnt, loff_t *fpos)
3661{
3662 char buf[64];
3663 const char *clockstr;
3664 int i;
3665
3666 if (cnt >= sizeof(buf))
3667 return -EINVAL;
3668
3669 if (copy_from_user(&buf, ubuf, cnt))
3670 return -EFAULT;
3671
3672 buf[cnt] = 0;
3673
3674 clockstr = strstrip(buf);
3675
3676 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3677 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3678 break;
3679 }
3680 if (i == ARRAY_SIZE(trace_clocks))
3681 return -EINVAL;
3682
3683 trace_clock_id = i;
3684
3685 mutex_lock(&trace_types_lock);
3686
3687 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3688 if (max_tr.buffer)
3689 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3690
3691 mutex_unlock(&trace_types_lock);
3692
3693 *fpos += cnt;
3694
3695 return cnt;
3696}
3697
3698static int tracing_clock_open(struct inode *inode, struct file *file)
3699{
3700 if (tracing_disabled)
3701 return -ENODEV;
3702 return single_open(file, tracing_clock_show, NULL);
3703}
3704
3705static const struct file_operations tracing_max_lat_fops = {
3706 .open = tracing_open_generic,
3707 .read = tracing_max_lat_read,
3708 .write = tracing_max_lat_write,
3709 .llseek = generic_file_llseek,
3710};
3711
3712static const struct file_operations tracing_ctrl_fops = {
3713 .open = tracing_open_generic,
3714 .read = tracing_ctrl_read,
3715 .write = tracing_ctrl_write,
3716 .llseek = generic_file_llseek,
3717};
3718
3719static const struct file_operations set_tracer_fops = {
3720 .open = tracing_open_generic,
3721 .read = tracing_set_trace_read,
3722 .write = tracing_set_trace_write,
3723 .llseek = generic_file_llseek,
3724};
3725
3726static const struct file_operations tracing_pipe_fops = {
3727 .open = tracing_open_pipe,
3728 .poll = tracing_poll_pipe,
3729 .read = tracing_read_pipe,
3730 .splice_read = tracing_splice_read_pipe,
3731 .release = tracing_release_pipe,
3732 .llseek = no_llseek,
3733};
3734
3735static const struct file_operations tracing_entries_fops = {
3736 .open = tracing_open_generic,
3737 .read = tracing_entries_read,
3738 .write = tracing_entries_write,
3739 .llseek = generic_file_llseek,
3740};
3741
3742static const struct file_operations tracing_free_buffer_fops = {
3743 .write = tracing_free_buffer_write,
3744 .release = tracing_free_buffer_release,
3745};
3746
3747static const struct file_operations tracing_mark_fops = {
3748 .open = tracing_open_generic,
3749 .write = tracing_mark_write,
3750 .llseek = generic_file_llseek,
3751};
3752
3753static const struct file_operations trace_clock_fops = {
3754 .open = tracing_clock_open,
3755 .read = seq_read,
3756 .llseek = seq_lseek,
3757 .release = single_release,
3758 .write = tracing_clock_write,
3759};
3760
3761struct ftrace_buffer_info {
3762 struct trace_array *tr;
3763 void *spare;
3764 int cpu;
3765 unsigned int read;
3766};
3767
3768static int tracing_buffers_open(struct inode *inode, struct file *filp)
3769{
3770 int cpu = (int)(long)inode->i_private;
3771 struct ftrace_buffer_info *info;
3772
3773 if (tracing_disabled)
3774 return -ENODEV;
3775
3776 info = kzalloc(sizeof(*info), GFP_KERNEL);
3777 if (!info)
3778 return -ENOMEM;
3779
3780 info->tr = &global_trace;
3781 info->cpu = cpu;
3782 info->spare = NULL;
3783 /* Force reading ring buffer for first read */
3784 info->read = (unsigned int)-1;
3785
3786 filp->private_data = info;
3787
3788 return nonseekable_open(inode, filp);
3789}
3790
3791static ssize_t
3792tracing_buffers_read(struct file *filp, char __user *ubuf,
3793 size_t count, loff_t *ppos)
3794{
3795 struct ftrace_buffer_info *info = filp->private_data;
3796 ssize_t ret;
3797 size_t size;
3798
3799 if (!count)
3800 return 0;
3801
3802 if (!info->spare)
3803 info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
3804 if (!info->spare)
3805 return -ENOMEM;
3806
3807 /* Do we have previous read data to read? */
3808 if (info->read < PAGE_SIZE)
3809 goto read;
3810
3811 info->read = 0;
3812
3813 trace_access_lock(info->cpu);
3814 ret = ring_buffer_read_page(info->tr->buffer,
3815 &info->spare,
3816 count,
3817 info->cpu, 0);
3818 trace_access_unlock(info->cpu);
3819 if (ret < 0)
3820 return 0;
3821
3822read:
3823 size = PAGE_SIZE - info->read;
3824 if (size > count)
3825 size = count;
3826
3827 ret = copy_to_user(ubuf, info->spare + info->read, size);
3828 if (ret == size)
3829 return -EFAULT;
3830 size -= ret;
3831
3832 *ppos += size;
3833 info->read += size;
3834
3835 return size;
3836}
3837
3838static int tracing_buffers_release(struct inode *inode, struct file *file)
3839{
3840 struct ftrace_buffer_info *info = file->private_data;
3841
3842 if (info->spare)
3843 ring_buffer_free_read_page(info->tr->buffer, info->spare);
3844 kfree(info);
3845
3846 return 0;
3847}
3848
3849struct buffer_ref {
3850 struct ring_buffer *buffer;
3851 void *page;
3852 int ref;
3853};
3854
3855static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3856 struct pipe_buffer *buf)
3857{
3858 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3859
3860 if (--ref->ref)
3861 return;
3862
3863 ring_buffer_free_read_page(ref->buffer, ref->page);
3864 kfree(ref);
3865 buf->private = 0;
3866}
3867
3868static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3869 struct pipe_buffer *buf)
3870{
3871 return 1;
3872}
3873
3874static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3875 struct pipe_buffer *buf)
3876{
3877 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3878
3879 ref->ref++;
3880}
3881
3882/* Pipe buffer operations for a buffer. */
3883static const struct pipe_buf_operations buffer_pipe_buf_ops = {
3884 .can_merge = 0,
3885 .map = generic_pipe_buf_map,
3886 .unmap = generic_pipe_buf_unmap,
3887 .confirm = generic_pipe_buf_confirm,
3888 .release = buffer_pipe_buf_release,
3889 .steal = buffer_pipe_buf_steal,
3890 .get = buffer_pipe_buf_get,
3891};
3892
3893/*
3894 * Callback from splice_to_pipe(), if we need to release some pages
3895 * at the end of the spd in case we error'ed out in filling the pipe.
3896 */
3897static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3898{
3899 struct buffer_ref *ref =
3900 (struct buffer_ref *)spd->partial[i].private;
3901
3902 if (--ref->ref)
3903 return;
3904
3905 ring_buffer_free_read_page(ref->buffer, ref->page);
3906 kfree(ref);
3907 spd->partial[i].private = 0;
3908}
3909
3910static ssize_t
3911tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3912 struct pipe_inode_info *pipe, size_t len,
3913 unsigned int flags)
3914{
3915 struct ftrace_buffer_info *info = file->private_data;
3916 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3917 struct page *pages_def[PIPE_DEF_BUFFERS];
3918 struct splice_pipe_desc spd = {
3919 .pages = pages_def,
3920 .partial = partial_def,
3921 .flags = flags,
3922 .ops = &buffer_pipe_buf_ops,
3923 .spd_release = buffer_spd_release,
3924 };
3925 struct buffer_ref *ref;
3926 int entries, size, i;
3927 size_t ret;
3928
3929 if (splice_grow_spd(pipe, &spd))
3930 return -ENOMEM;
3931
3932 if (*ppos & (PAGE_SIZE - 1)) {
3933 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3934 ret = -EINVAL;
3935 goto out;
3936 }
3937
3938 if (len & (PAGE_SIZE - 1)) {
3939 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3940 if (len < PAGE_SIZE) {
3941 ret = -EINVAL;
3942 goto out;
3943 }
3944 len &= PAGE_MASK;
3945 }
3946
3947 trace_access_lock(info->cpu);
3948 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3949
3950 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
3951 struct page *page;
3952 int r;
3953
3954 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3955 if (!ref)
3956 break;
3957
3958 ref->ref = 1;
3959 ref->buffer = info->tr->buffer;
3960 ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
3961 if (!ref->page) {
3962 kfree(ref);
3963 break;
3964 }
3965
3966 r = ring_buffer_read_page(ref->buffer, &ref->page,
3967 len, info->cpu, 1);
3968 if (r < 0) {
3969 ring_buffer_free_read_page(ref->buffer, ref->page);
3970 kfree(ref);
3971 break;
3972 }
3973
3974 /*
3975 * zero out any left over data, this is going to
3976 * user land.
3977 */
3978 size = ring_buffer_page_len(ref->page);
3979 if (size < PAGE_SIZE)
3980 memset(ref->page + size, 0, PAGE_SIZE - size);
3981
3982 page = virt_to_page(ref->page);
3983
3984 spd.pages[i] = page;
3985 spd.partial[i].len = PAGE_SIZE;
3986 spd.partial[i].offset = 0;
3987 spd.partial[i].private = (unsigned long)ref;
3988 spd.nr_pages++;
3989 *ppos += PAGE_SIZE;
3990
3991 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3992 }
3993
3994 trace_access_unlock(info->cpu);
3995 spd.nr_pages = i;
3996
3997 /* did we read anything? */
3998 if (!spd.nr_pages) {
3999 if (flags & SPLICE_F_NONBLOCK)
4000 ret = -EAGAIN;
4001 else
4002 ret = 0;
4003 /* TODO: block */
4004 goto out;
4005 }
4006
4007 ret = splice_to_pipe(pipe, &spd);
4008 splice_shrink_spd(pipe, &spd);
4009out:
4010 return ret;
4011}
4012
4013static const struct file_operations tracing_buffers_fops = {
4014 .open = tracing_buffers_open,
4015 .read = tracing_buffers_read,
4016 .release = tracing_buffers_release,
4017 .splice_read = tracing_buffers_splice_read,
4018 .llseek = no_llseek,
4019};
4020
4021static ssize_t
4022tracing_stats_read(struct file *filp, char __user *ubuf,
4023 size_t count, loff_t *ppos)
4024{
4025 unsigned long cpu = (unsigned long)filp->private_data;
4026 struct trace_array *tr = &global_trace;
4027 struct trace_seq *s;
4028 unsigned long cnt;
4029
4030 s = kmalloc(sizeof(*s), GFP_KERNEL);
4031 if (!s)
4032 return -ENOMEM;
4033
4034 trace_seq_init(s);
4035
4036 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4037 trace_seq_printf(s, "entries: %ld\n", cnt);
4038
4039 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4040 trace_seq_printf(s, "overrun: %ld\n", cnt);
4041
4042 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4043 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4044
4045 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4046
4047 kfree(s);
4048
4049 return count;
4050}
4051
4052static const struct file_operations tracing_stats_fops = {
4053 .open = tracing_open_generic,
4054 .read = tracing_stats_read,
4055 .llseek = generic_file_llseek,
4056};
4057
4058#ifdef CONFIG_DYNAMIC_FTRACE
4059
4060int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4061{
4062 return 0;
4063}
4064
4065static ssize_t
4066tracing_read_dyn_info(struct file *filp, char __user *ubuf,
4067 size_t cnt, loff_t *ppos)
4068{
4069 static char ftrace_dyn_info_buffer[1024];
4070 static DEFINE_MUTEX(dyn_info_mutex);
4071 unsigned long *p = filp->private_data;
4072 char *buf = ftrace_dyn_info_buffer;
4073 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4074 int r;
4075
4076 mutex_lock(&dyn_info_mutex);
4077 r = sprintf(buf, "%ld ", *p);
4078
4079 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4080 buf[r++] = '\n';
4081
4082 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4083
4084 mutex_unlock(&dyn_info_mutex);
4085
4086 return r;
4087}
4088
4089static const struct file_operations tracing_dyn_info_fops = {
4090 .open = tracing_open_generic,
4091 .read = tracing_read_dyn_info,
4092 .llseek = generic_file_llseek,
4093};
4094#endif
4095
4096static struct dentry *d_tracer;
4097
4098struct dentry *tracing_init_dentry(void)
4099{
4100 static int once;
4101
4102 if (d_tracer)
4103 return d_tracer;
4104
4105 if (!debugfs_initialized())
4106 return NULL;
4107
4108 d_tracer = debugfs_create_dir("tracing", NULL);
4109
4110 if (!d_tracer && !once) {
4111 once = 1;
4112 pr_warning("Could not create debugfs directory 'tracing'\n");
4113 return NULL;
4114 }
4115
4116 return d_tracer;
4117}
4118
4119static struct dentry *d_percpu;
4120
4121struct dentry *tracing_dentry_percpu(void)
4122{
4123 static int once;
4124 struct dentry *d_tracer;
4125
4126 if (d_percpu)
4127 return d_percpu;
4128
4129 d_tracer = tracing_init_dentry();
4130
4131 if (!d_tracer)
4132 return NULL;
4133
4134 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4135
4136 if (!d_percpu && !once) {
4137 once = 1;
4138 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4139 return NULL;
4140 }
4141
4142 return d_percpu;
4143}
4144
4145static void tracing_init_debugfs_percpu(long cpu)
4146{
4147 struct dentry *d_percpu = tracing_dentry_percpu();
4148 struct dentry *d_cpu;
4149 char cpu_dir[30]; /* 30 characters should be more than enough */
4150
4151 snprintf(cpu_dir, 30, "cpu%ld", cpu);
4152 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4153 if (!d_cpu) {
4154 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4155 return;
4156 }
4157
4158 /* per cpu trace_pipe */
4159 trace_create_file("trace_pipe", 0444, d_cpu,
4160 (void *) cpu, &tracing_pipe_fops);
4161
4162 /* per cpu trace */
4163 trace_create_file("trace", 0644, d_cpu,
4164 (void *) cpu, &tracing_fops);
4165
4166 trace_create_file("trace_pipe_raw", 0444, d_cpu,
4167 (void *) cpu, &tracing_buffers_fops);
4168
4169 trace_create_file("stats", 0444, d_cpu,
4170 (void *) cpu, &tracing_stats_fops);
4171}
4172
4173#ifdef CONFIG_FTRACE_SELFTEST
4174/* Let selftest have access to static functions in this file */
4175#include "trace_selftest.c"
4176#endif
4177
4178struct trace_option_dentry {
4179 struct tracer_opt *opt;
4180 struct tracer_flags *flags;
4181 struct dentry *entry;
4182};
4183
4184static ssize_t
4185trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4186 loff_t *ppos)
4187{
4188 struct trace_option_dentry *topt = filp->private_data;
4189 char *buf;
4190
4191 if (topt->flags->val & topt->opt->bit)
4192 buf = "1\n";
4193 else
4194 buf = "0\n";
4195
4196 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4197}
4198
4199static ssize_t
4200trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4201 loff_t *ppos)
4202{
4203 struct trace_option_dentry *topt = filp->private_data;
4204 unsigned long val;
4205 int ret;
4206
4207 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4208 if (ret)
4209 return ret;
4210
4211 if (val != 0 && val != 1)
4212 return -EINVAL;
4213
4214 if (!!(topt->flags->val & topt->opt->bit) != val) {
4215 mutex_lock(&trace_types_lock);
4216 ret = __set_tracer_option(current_trace, topt->flags,
4217 topt->opt, !val);
4218 mutex_unlock(&trace_types_lock);
4219 if (ret)
4220 return ret;
4221 }
4222
4223 *ppos += cnt;
4224
4225 return cnt;
4226}
4227
4228
4229static const struct file_operations trace_options_fops = {
4230 .open = tracing_open_generic,
4231 .read = trace_options_read,
4232 .write = trace_options_write,
4233 .llseek = generic_file_llseek,
4234};
4235
4236static ssize_t
4237trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4238 loff_t *ppos)
4239{
4240 long index = (long)filp->private_data;
4241 char *buf;
4242
4243 if (trace_flags & (1 << index))
4244 buf = "1\n";
4245 else
4246 buf = "0\n";
4247
4248 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4249}
4250
4251static ssize_t
4252trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4253 loff_t *ppos)
4254{
4255 long index = (long)filp->private_data;
4256 unsigned long val;
4257 int ret;
4258
4259 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4260 if (ret)
4261 return ret;
4262
4263 if (val != 0 && val != 1)
4264 return -EINVAL;
4265 set_tracer_flags(1 << index, val);
4266
4267 *ppos += cnt;
4268
4269 return cnt;
4270}
4271
4272static const struct file_operations trace_options_core_fops = {
4273 .open = tracing_open_generic,
4274 .read = trace_options_core_read,
4275 .write = trace_options_core_write,
4276 .llseek = generic_file_llseek,
4277};
4278
4279struct dentry *trace_create_file(const char *name,
4280 mode_t mode,
4281 struct dentry *parent,
4282 void *data,
4283 const struct file_operations *fops)
4284{
4285 struct dentry *ret;
4286
4287 ret = debugfs_create_file(name, mode, parent, data, fops);
4288 if (!ret)
4289 pr_warning("Could not create debugfs '%s' entry\n", name);
4290
4291 return ret;
4292}
4293
4294
4295static struct dentry *trace_options_init_dentry(void)
4296{
4297 struct dentry *d_tracer;
4298 static struct dentry *t_options;
4299
4300 if (t_options)
4301 return t_options;
4302
4303 d_tracer = tracing_init_dentry();
4304 if (!d_tracer)
4305 return NULL;
4306
4307 t_options = debugfs_create_dir("options", d_tracer);
4308 if (!t_options) {
4309 pr_warning("Could not create debugfs directory 'options'\n");
4310 return NULL;
4311 }
4312
4313 return t_options;
4314}
4315
4316static void
4317create_trace_option_file(struct trace_option_dentry *topt,
4318 struct tracer_flags *flags,
4319 struct tracer_opt *opt)
4320{
4321 struct dentry *t_options;
4322
4323 t_options = trace_options_init_dentry();
4324 if (!t_options)
4325 return;
4326
4327 topt->flags = flags;
4328 topt->opt = opt;
4329
4330 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4331 &trace_options_fops);
4332
4333}
4334
4335static struct trace_option_dentry *
4336create_trace_option_files(struct tracer *tracer)
4337{
4338 struct trace_option_dentry *topts;
4339 struct tracer_flags *flags;
4340 struct tracer_opt *opts;
4341 int cnt;
4342
4343 if (!tracer)
4344 return NULL;
4345
4346 flags = tracer->flags;
4347
4348 if (!flags || !flags->opts)
4349 return NULL;
4350
4351 opts = flags->opts;
4352
4353 for (cnt = 0; opts[cnt].name; cnt++)
4354 ;
4355
4356 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4357 if (!topts)
4358 return NULL;
4359
4360 for (cnt = 0; opts[cnt].name; cnt++)
4361 create_trace_option_file(&topts[cnt], flags,
4362 &opts[cnt]);
4363
4364 return topts;
4365}
4366
4367static void
4368destroy_trace_option_files(struct trace_option_dentry *topts)
4369{
4370 int cnt;
4371
4372 if (!topts)
4373 return;
4374
4375 for (cnt = 0; topts[cnt].opt; cnt++) {
4376 if (topts[cnt].entry)
4377 debugfs_remove(topts[cnt].entry);
4378 }
4379
4380 kfree(topts);
4381}
4382
4383static struct dentry *
4384create_trace_option_core_file(const char *option, long index)
4385{
4386 struct dentry *t_options;
4387
4388 t_options = trace_options_init_dentry();
4389 if (!t_options)
4390 return NULL;
4391
4392 return trace_create_file(option, 0644, t_options, (void *)index,
4393 &trace_options_core_fops);
4394}
4395
4396static __init void create_trace_options_dir(void)
4397{
4398 struct dentry *t_options;
4399 int i;
4400
4401 t_options = trace_options_init_dentry();
4402 if (!t_options)
4403 return;
4404
4405 for (i = 0; trace_options[i]; i++)
4406 create_trace_option_core_file(trace_options[i], i);
4407}
4408
4409static __init int tracer_init_debugfs(void)
4410{
4411 struct dentry *d_tracer;
4412 int cpu;
4413
4414 trace_access_lock_init();
4415
4416 d_tracer = tracing_init_dentry();
4417
4418 trace_create_file("tracing_enabled", 0644, d_tracer,
4419 &global_trace, &tracing_ctrl_fops);
4420
4421 trace_create_file("trace_options", 0644, d_tracer,
4422 NULL, &tracing_iter_fops);
4423
4424 trace_create_file("tracing_cpumask", 0644, d_tracer,
4425 NULL, &tracing_cpumask_fops);
4426
4427 trace_create_file("trace", 0644, d_tracer,
4428 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4429
4430 trace_create_file("available_tracers", 0444, d_tracer,
4431 &global_trace, &show_traces_fops);
4432
4433 trace_create_file("current_tracer", 0644, d_tracer,
4434 &global_trace, &set_tracer_fops);
4435
4436#ifdef CONFIG_TRACER_MAX_TRACE
4437 trace_create_file("tracing_max_latency", 0644, d_tracer,
4438 &tracing_max_latency, &tracing_max_lat_fops);
4439#endif
4440
4441 trace_create_file("tracing_thresh", 0644, d_tracer,
4442 &tracing_thresh, &tracing_max_lat_fops);
4443
4444 trace_create_file("README", 0444, d_tracer,
4445 NULL, &tracing_readme_fops);
4446
4447 trace_create_file("trace_pipe", 0444, d_tracer,
4448 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4449
4450 trace_create_file("buffer_size_kb", 0644, d_tracer,
4451 &global_trace, &tracing_entries_fops);
4452
4453 trace_create_file("free_buffer", 0644, d_tracer,
4454 &global_trace, &tracing_free_buffer_fops);
4455
4456 trace_create_file("trace_marker", 0220, d_tracer,
4457 NULL, &tracing_mark_fops);
4458
4459 trace_create_file("saved_cmdlines", 0444, d_tracer,
4460 NULL, &tracing_saved_cmdlines_fops);
4461
4462 trace_create_file("trace_clock", 0644, d_tracer, NULL,
4463 &trace_clock_fops);
4464
4465#ifdef CONFIG_DYNAMIC_FTRACE
4466 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4467 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4468#endif
4469
4470 create_trace_options_dir();
4471
4472 for_each_tracing_cpu(cpu)
4473 tracing_init_debugfs_percpu(cpu);
4474
4475 return 0;
4476}
4477
4478static int trace_panic_handler(struct notifier_block *this,
4479 unsigned long event, void *unused)
4480{
4481 if (ftrace_dump_on_oops)
4482 ftrace_dump(ftrace_dump_on_oops);
4483 return NOTIFY_OK;
4484}
4485
4486static struct notifier_block trace_panic_notifier = {
4487 .notifier_call = trace_panic_handler,
4488 .next = NULL,
4489 .priority = 150 /* priority: INT_MAX >= x >= 0 */
4490};
4491
4492static int trace_die_handler(struct notifier_block *self,
4493 unsigned long val,
4494 void *data)
4495{
4496 switch (val) {
4497 case DIE_OOPS:
4498 if (ftrace_dump_on_oops)
4499 ftrace_dump(ftrace_dump_on_oops);
4500 break;
4501 default:
4502 break;
4503 }
4504 return NOTIFY_OK;
4505}
4506
4507static struct notifier_block trace_die_notifier = {
4508 .notifier_call = trace_die_handler,
4509 .priority = 200
4510};
4511
4512/*
4513 * printk is set to max of 1024, we really don't need it that big.
4514 * Nothing should be printing 1000 characters anyway.
4515 */
4516#define TRACE_MAX_PRINT 1000
4517
4518/*
4519 * Define here KERN_TRACE so that we have one place to modify
4520 * it if we decide to change what log level the ftrace dump
4521 * should be at.
4522 */
4523#define KERN_TRACE KERN_EMERG
4524
4525void
4526trace_printk_seq(struct trace_seq *s)
4527{
4528 /* Probably should print a warning here. */
4529 if (s->len >= 1000)
4530 s->len = 1000;
4531
4532 /* should be zero ended, but we are paranoid. */
4533 s->buffer[s->len] = 0;
4534
4535 printk(KERN_TRACE "%s", s->buffer);
4536
4537 trace_seq_init(s);
4538}
4539
4540void trace_init_global_iter(struct trace_iterator *iter)
4541{
4542 iter->tr = &global_trace;
4543 iter->trace = current_trace;
4544 iter->cpu_file = TRACE_PIPE_ALL_CPU;
4545}
4546
4547static void
4548__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4549{
4550 static arch_spinlock_t ftrace_dump_lock =
4551 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4552 /* use static because iter can be a bit big for the stack */
4553 static struct trace_iterator iter;
4554 unsigned int old_userobj;
4555 static int dump_ran;
4556 unsigned long flags;
4557 int cnt = 0, cpu;
4558
4559 /* only one dump */
4560 local_irq_save(flags);
4561 arch_spin_lock(&ftrace_dump_lock);
4562 if (dump_ran)
4563 goto out;
4564
4565 dump_ran = 1;
4566
4567 tracing_off();
4568
4569 if (disable_tracing)
4570 ftrace_kill();
4571
4572 trace_init_global_iter(&iter);
4573
4574 for_each_tracing_cpu(cpu) {
4575 atomic_inc(&iter.tr->data[cpu]->disabled);
4576 }
4577
4578 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4579
4580 /* don't look at user memory in panic mode */
4581 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4582
4583 /* Simulate the iterator */
4584 iter.tr = &global_trace;
4585 iter.trace = current_trace;
4586
4587 switch (oops_dump_mode) {
4588 case DUMP_ALL:
4589 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4590 break;
4591 case DUMP_ORIG:
4592 iter.cpu_file = raw_smp_processor_id();
4593 break;
4594 case DUMP_NONE:
4595 goto out_enable;
4596 default:
4597 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4598 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4599 }
4600
4601 printk(KERN_TRACE "Dumping ftrace buffer:\n");
4602
4603 /*
4604 * We need to stop all tracing on all CPUS to read the
4605 * the next buffer. This is a bit expensive, but is
4606 * not done often. We fill all what we can read,
4607 * and then release the locks again.
4608 */
4609
4610 while (!trace_empty(&iter)) {
4611
4612 if (!cnt)
4613 printk(KERN_TRACE "---------------------------------\n");
4614
4615 cnt++;
4616
4617 /* reset all but tr, trace, and overruns */
4618 memset(&iter.seq, 0,
4619 sizeof(struct trace_iterator) -
4620 offsetof(struct trace_iterator, seq));
4621 iter.iter_flags |= TRACE_FILE_LAT_FMT;
4622 iter.pos = -1;
4623
4624 if (trace_find_next_entry_inc(&iter) != NULL) {
4625 int ret;
4626
4627 ret = print_trace_line(&iter);
4628 if (ret != TRACE_TYPE_NO_CONSUME)
4629 trace_consume(&iter);
4630 }
4631
4632 trace_printk_seq(&iter.seq);
4633 }
4634
4635 if (!cnt)
4636 printk(KERN_TRACE " (ftrace buffer empty)\n");
4637 else
4638 printk(KERN_TRACE "---------------------------------\n");
4639
4640 out_enable:
4641 /* Re-enable tracing if requested */
4642 if (!disable_tracing) {
4643 trace_flags |= old_userobj;
4644
4645 for_each_tracing_cpu(cpu) {
4646 atomic_dec(&iter.tr->data[cpu]->disabled);
4647 }
4648 tracing_on();
4649 }
4650
4651 out:
4652 arch_spin_unlock(&ftrace_dump_lock);
4653 local_irq_restore(flags);
4654}
4655
4656/* By default: disable tracing after the dump */
4657void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
4658{
4659 __ftrace_dump(true, oops_dump_mode);
4660}
4661
4662__init static int tracer_alloc_buffers(void)
4663{
4664 int ring_buf_size;
4665 enum ring_buffer_flags rb_flags;
4666 int i;
4667 int ret = -ENOMEM;
4668
4669
4670 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4671 goto out;
4672
4673 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4674 goto out_free_buffer_mask;
4675
4676 /* To save memory, keep the ring buffer size to its minimum */
4677 if (ring_buffer_expanded)
4678 ring_buf_size = trace_buf_size;
4679 else
4680 ring_buf_size = 1;
4681
4682 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
4683
4684 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4685 cpumask_copy(tracing_cpumask, cpu_all_mask);
4686
4687 /* TODO: make the number of buffers hot pluggable with CPUS */
4688 global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
4689 if (!global_trace.buffer) {
4690 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4691 WARN_ON(1);
4692 goto out_free_cpumask;
4693 }
4694 global_trace.entries = ring_buffer_size(global_trace.buffer);
4695
4696
4697#ifdef CONFIG_TRACER_MAX_TRACE
4698 max_tr.buffer = ring_buffer_alloc(1, rb_flags);
4699 if (!max_tr.buffer) {
4700 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4701 WARN_ON(1);
4702 ring_buffer_free(global_trace.buffer);
4703 goto out_free_cpumask;
4704 }
4705 max_tr.entries = 1;
4706#endif
4707
4708 /* Allocate the first page for all buffers */
4709 for_each_tracing_cpu(i) {
4710 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
4711 max_tr.data[i] = &per_cpu(max_tr_data, i);
4712 }
4713
4714 trace_init_cmdlines();
4715
4716 register_tracer(&nop_trace);
4717 current_trace = &nop_trace;
4718 /* All seems OK, enable tracing */
4719 tracing_disabled = 0;
4720
4721 atomic_notifier_chain_register(&panic_notifier_list,
4722 &trace_panic_notifier);
4723
4724 register_die_notifier(&trace_die_notifier);
4725
4726 return 0;
4727
4728out_free_cpumask:
4729 free_cpumask_var(tracing_cpumask);
4730out_free_buffer_mask:
4731 free_cpumask_var(tracing_buffer_mask);
4732out:
4733 return ret;
4734}
4735
4736__init static int clear_boot_tracer(void)
4737{
4738 /*
4739 * The default tracer at boot buffer is an init section.
4740 * This function is called in lateinit. If we did not
4741 * find the boot tracer, then clear it out, to prevent
4742 * later registration from accessing the buffer that is
4743 * about to be freed.
4744 */
4745 if (!default_bootup_tracer)
4746 return 0;
4747
4748 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4749 default_bootup_tracer);
4750 default_bootup_tracer = NULL;
4751
4752 return 0;
4753}
4754
4755early_initcall(tracer_alloc_buffers);
4756fs_initcall(tracer_init_debugfs);
4757late_initcall(clear_boot_tracer);