1/*
   2 * ring buffer based function tracer
   3 *
   4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6 *
   7 * Originally taken from the RT patch by:
   8 *    Arnaldo Carvalho de Melo <acme@redhat.com>
   9 *
  10 * Based on code from the latency_tracer, that is:
  11 *  Copyright (C) 2004-2006 Ingo Molnar
  12 *  Copyright (C) 2004 William Lee Irwin III
  13 */
  14#include <linux/ring_buffer.h>
  15#include <generated/utsrelease.h>
  16#include <linux/stacktrace.h>
  17#include <linux/writeback.h>
  18#include <linux/kallsyms.h>
 
  19#include <linux/seq_file.h>
  20#include <linux/notifier.h>
  21#include <linux/irqflags.h>
  22#include <linux/debugfs.h>
 
  23#include <linux/pagemap.h>
  24#include <linux/hardirq.h>
  25#include <linux/linkage.h>
  26#include <linux/uaccess.h>
  27#include <linux/kprobes.h>
  28#include <linux/ftrace.h>
  29#include <linux/module.h>
  30#include <linux/percpu.h>
  31#include <linux/splice.h>
  32#include <linux/kdebug.h>
  33#include <linux/string.h>
 
  34#include <linux/rwsem.h>
  35#include <linux/slab.h>
  36#include <linux/ctype.h>
  37#include <linux/init.h>
 
  38#include <linux/poll.h>
  39#include <linux/nmi.h>
  40#include <linux/fs.h>
 
 
 
 
 
 
 
 
  41
  42#include "trace.h"
  43#include "trace_output.h"
  44
  45/*
  46 * On boot up, the ring buffer is set to the minimum size, so that
  47 * we do not waste memory on systems that are not using tracing.
  48 */
  49int ring_buffer_expanded;
  50
  51/*
  52 * We need to change this state when a selftest is running.
  53 * A selftest will lurk into the ring-buffer to count the
  54 * entries inserted during the selftest although some concurrent
  55 * insertions into the ring-buffer such as trace_printk could occurred
  56 * at the same time, giving false positive or negative results.
  57 */
  58static bool __read_mostly tracing_selftest_running;
  59
  60/*
  61 * If a tracer is running, we do not want to run SELFTEST.
 
  62 */
  63bool __read_mostly tracing_selftest_disabled;
  64
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  65/* For tracers that don't implement custom flags */
  66static struct tracer_opt dummy_tracer_opt[] = {
  67	{ }
  68};
  69
  70static struct tracer_flags dummy_tracer_flags = {
  71	.val = 0,
  72	.opts = dummy_tracer_opt
  73};
  74
  75static int dummy_set_flag(u32 old_flags, u32 bit, int set)
  76{
  77	return 0;
  78}
  79
  80/*
 
 
 
 
 
 
 
  81 * Kill all tracing for good (never come back).
  82 * It is initialized to 1 but will turn to zero if the initialization
  83 * of the tracer is successful. But that is the only place that sets
  84 * this back to zero.
  85 */
  86static int tracing_disabled = 1;
  87
  88DEFINE_PER_CPU(int, ftrace_cpu_disabled);
  89
  90cpumask_var_t __read_mostly	tracing_buffer_mask;
  91
  92/*
  93 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
  94 *
  95 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
  96 * is set, then ftrace_dump is called. This will output the contents
  97 * of the ftrace buffers to the console.  This is very useful for
  98 * capturing traces that lead to crashes and outputing it to a
  99 * serial console.
 100 *
 101 * It is default off, but you can enable it with either specifying
 102 * "ftrace_dump_on_oops" in the kernel command line, or setting
 103 * /proc/sys/kernel/ftrace_dump_on_oops
 104 * Set 1 if you want to dump buffers of all CPUs
 105 * Set 2 if you want to dump the buffer of the CPU that triggered oops
 
 
 
 106 */
 
 
 
 
 
 
 
 
 
 
 
 
 107
 108enum ftrace_dump_mode ftrace_dump_on_oops;
 109
 110static int tracing_set_tracer(const char *buf);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 111
 112#define MAX_TRACER_SIZE		100
 113static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
 114static char *default_bootup_tracer;
 115
 
 
 
 
 
 
 
 
 
 116static int __init set_cmdline_ftrace(char *str)
 117{
 118	strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
 119	default_bootup_tracer = bootup_tracer_buf;
 120	/* We are using ftrace early, expand it */
 121	ring_buffer_expanded = 1;
 122	return 1;
 123}
 124__setup("ftrace=", set_cmdline_ftrace);
 125
 
 
 
 
 
 
 
 
 126static int __init set_ftrace_dump_on_oops(char *str)
 127{
 128	if (*str++ != '=' || !*str) {
 129		ftrace_dump_on_oops = DUMP_ALL;
 130		return 1;
 131	}
 132
 133	if (!strcmp("orig_cpu", str)) {
 134		ftrace_dump_on_oops = DUMP_ORIG;
 135                return 1;
 136        }
 
 
 
 
 
 
 137
 138        return 0;
 139}
 140__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
 141
 142unsigned long long ns2usecs(cycle_t nsec)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 143{
 144	nsec += 500;
 145	do_div(nsec, 1000);
 146	return nsec;
 147}
 148
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 149/*
 150 * The global_trace is the descriptor that holds the tracing
 151 * buffers for the live tracing. For each CPU, it contains
 152 * a link list of pages that will store trace entries. The
 153 * page descriptor of the pages in the memory is used to hold
 154 * the link list by linking the lru item in the page descriptor
 155 * to each of the pages in the buffer per CPU.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 156 *
 157 * For each active CPU there is a data field that holds the
 158 * pages for the buffer for that CPU. Each CPU has the same number
 159 * of pages allocated for its buffer.
 160 */
 161static struct trace_array	global_trace;
 
 
 
 162
 163static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
 
 
 
 
 164
 165int filter_current_check_discard(struct ring_buffer *buffer,
 166				 struct ftrace_event_call *call, void *rec,
 167				 struct ring_buffer_event *event)
 168{
 169	return filter_check_discard(call, rec, buffer, event);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 170}
 171EXPORT_SYMBOL_GPL(filter_current_check_discard);
 172
 173cycle_t ftrace_now(int cpu)
 
 
 
 
 
 
 
 
 174{
 175	u64 ts;
 
 176
 177	/* Early boot up does not have a buffer yet */
 178	if (!global_trace.buffer)
 179		return trace_clock_local();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 180
 181	ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
 182	ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 183
 184	return ts;
 
 
 
 
 
 
 
 
 
 
 185}
 186
 187/*
 188 * The max_tr is used to snapshot the global_trace when a maximum
 189 * latency is reached. Some tracers will use this to store a maximum
 190 * trace while it continues examining live traces.
 
 
 
 
 191 *
 192 * The buffers for the max_tr are set up the same as the global_trace.
 193 * When a snapshot is taken, the link list of the max_tr is swapped
 194 * with the link list of the global_trace and the buffers are reset for
 195 * the global_trace so the tracing can continue.
 196 */
 197static struct trace_array	max_tr;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 198
 199static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 200
 201/* tracer_enabled is used to toggle activation of a tracer */
 202static int			tracer_enabled = 1;
 
 
 203
 204/**
 205 * tracing_is_enabled - return tracer_enabled status
 206 *
 207 * This function is used by other tracers to know the status
 208 * of the tracer_enabled flag.  Tracers may use this function
 209 * to know if it should enable their features when starting
 210 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
 
 211 */
 212int tracing_is_enabled(void)
 213{
 214	return tracer_enabled;
 
 
 
 
 
 
 215}
 216
 217/*
 218 * trace_buf_size is the size in bytes that is allocated
 219 * for a buffer. Note, the number of bytes is always rounded
 220 * to page size.
 221 *
 222 * This number is purposely set to a low number of 16384.
 223 * If the dump on oops happens, it will be much appreciated
 224 * to not have to wait for all that output. Anyway this can be
 225 * boot time and run time configurable.
 226 */
 227#define TRACE_BUF_SIZE_DEFAULT	1441792UL /* 16384 * 88 (sizeof(entry)) */
 228
 229static unsigned long		trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 230
 231/* trace_types holds a link list of available tracers. */
 232static struct tracer		*trace_types __read_mostly;
 233
 234/* current_trace points to the tracer that is currently active */
 235static struct tracer		*current_trace __read_mostly;
 236
 237/*
 238 * trace_types_lock is used to protect the trace_types list.
 239 */
 240static DEFINE_MUTEX(trace_types_lock);
 241
 242/*
 243 * serialize the access of the ring buffer
 244 *
 245 * ring buffer serializes readers, but it is low level protection.
 246 * The validity of the events (which returns by ring_buffer_peek() ..etc)
 247 * are not protected by ring buffer.
 248 *
 249 * The content of events may become garbage if we allow other process consumes
 250 * these events concurrently:
 251 *   A) the page of the consumed events may become a normal page
 252 *      (not reader page) in ring buffer, and this page will be rewrited
 253 *      by events producer.
 254 *   B) The page of the consumed events may become a page for splice_read,
 255 *      and this page will be returned to system.
 256 *
 257 * These primitives allow multi process access to different cpu ring buffer
 258 * concurrently.
 259 *
 260 * These primitives don't distinguish read-only and read-consume access.
 261 * Multi read-only access are also serialized.
 262 */
 263
 264#ifdef CONFIG_SMP
 265static DECLARE_RWSEM(all_cpu_access_lock);
 266static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
 267
 268static inline void trace_access_lock(int cpu)
 269{
 270	if (cpu == TRACE_PIPE_ALL_CPU) {
 271		/* gain it for accessing the whole ring buffer. */
 272		down_write(&all_cpu_access_lock);
 273	} else {
 274		/* gain it for accessing a cpu ring buffer. */
 275
 276		/* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
 277		down_read(&all_cpu_access_lock);
 278
 279		/* Secondly block other access to this @cpu ring buffer. */
 280		mutex_lock(&per_cpu(cpu_access_lock, cpu));
 281	}
 282}
 283
 284static inline void trace_access_unlock(int cpu)
 285{
 286	if (cpu == TRACE_PIPE_ALL_CPU) {
 287		up_write(&all_cpu_access_lock);
 288	} else {
 289		mutex_unlock(&per_cpu(cpu_access_lock, cpu));
 290		up_read(&all_cpu_access_lock);
 291	}
 292}
 293
 294static inline void trace_access_lock_init(void)
 295{
 296	int cpu;
 297
 298	for_each_possible_cpu(cpu)
 299		mutex_init(&per_cpu(cpu_access_lock, cpu));
 300}
 301
 302#else
 303
 304static DEFINE_MUTEX(access_lock);
 305
 306static inline void trace_access_lock(int cpu)
 307{
 308	(void)cpu;
 309	mutex_lock(&access_lock);
 310}
 311
 312static inline void trace_access_unlock(int cpu)
 313{
 314	(void)cpu;
 315	mutex_unlock(&access_lock);
 316}
 317
 318static inline void trace_access_lock_init(void)
 319{
 320}
 321
 322#endif
 323
 324/* trace_wait is a waitqueue for tasks blocked on trace_poll */
 325static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
 
 
 
 
 
 
 326
 327/* trace_flags holds trace_options default values */
 328unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
 329	TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
 330	TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
 331	TRACE_ITER_IRQ_INFO;
 
 
 
 
 
 
 
 332
 333static int trace_stop_count;
 334static DEFINE_RAW_SPINLOCK(tracing_start_lock);
 335
 336static void wakeup_work_handler(struct work_struct *work)
 
 
 337{
 338	wake_up(&trace_wait);
 
 
 339}
 340
 341static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 342
 343/**
 344 * tracing_on - enable tracing buffers
 345 *
 346 * This function enables tracing buffers that may have been
 347 * disabled with tracing_off.
 348 */
 349void tracing_on(void)
 350{
 351	if (global_trace.buffer)
 352		ring_buffer_record_on(global_trace.buffer);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 353	/*
 354	 * This flag is only looked at when buffers haven't been
 355	 * allocated yet. We don't really care about the race
 356	 * between setting this flag and actually turning
 357	 * on the buffer.
 358	 */
 359	global_trace.buffer_disabled = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 360}
 361EXPORT_SYMBOL_GPL(tracing_on);
 362
 363/**
 364 * tracing_off - turn off tracing buffers
 365 *
 366 * This function stops the tracing buffers from recording data.
 367 * It does not disable any overhead the tracers themselves may
 368 * be causing. This function simply causes all recording to
 369 * the ring buffers to fail.
 370 */
 371void tracing_off(void)
 372{
 373	if (global_trace.buffer)
 374		ring_buffer_record_off(global_trace.buffer);
 375	/*
 376	 * This flag is only looked at when buffers haven't been
 377	 * allocated yet. We don't really care about the race
 378	 * between setting this flag and actually turning
 379	 * on the buffer.
 380	 */
 381	global_trace.buffer_disabled = 1;
 382}
 383EXPORT_SYMBOL_GPL(tracing_off);
 384
 385/**
 386 * tracing_is_on - show state of ring buffers enabled
 387 */
 388int tracing_is_on(void)
 389{
 390	if (global_trace.buffer)
 391		return ring_buffer_record_is_on(global_trace.buffer);
 392	return !global_trace.buffer_disabled;
 
 
 393}
 394EXPORT_SYMBOL_GPL(tracing_is_on);
 395
 396/**
 397 * trace_wake_up - wake up tasks waiting for trace input
 
 398 *
 399 * Schedules a delayed work to wake up any task that is blocked on the
 400 * trace_wait queue. These is used with trace_poll for tasks polling the
 401 * trace.
 402 */
 403void trace_wake_up(void)
 404{
 405	const unsigned long delay = msecs_to_jiffies(2);
 
 
 
 406
 407	if (trace_flags & TRACE_ITER_BLOCK)
 408		return;
 409	schedule_delayed_work(&wakeup_work, delay);
 
 
 
 410}
 
 411
 412static int __init set_buf_size(char *str)
 413{
 414	unsigned long buf_size;
 415
 416	if (!str)
 417		return 0;
 418	buf_size = memparse(str, &str);
 419	/* nr_entries can not be zero */
 420	if (buf_size == 0)
 421		return 0;
 422	trace_buf_size = buf_size;
 
 
 423	return 1;
 424}
 425__setup("trace_buf_size=", set_buf_size);
 426
 427static int __init set_tracing_thresh(char *str)
 428{
 429	unsigned long threshhold;
 430	int ret;
 431
 432	if (!str)
 433		return 0;
 434	ret = strict_strtoul(str, 0, &threshhold);
 435	if (ret < 0)
 436		return 0;
 437	tracing_thresh = threshhold * 1000;
 438	return 1;
 439}
 440__setup("tracing_thresh=", set_tracing_thresh);
 441
 442unsigned long nsecs_to_usecs(unsigned long nsecs)
 443{
 444	return nsecs / 1000;
 445}
 446
 447/* These must match the bit postions in trace_iterator_flags */
 
 
 
 
 
 
 
 
 
 448static const char *trace_options[] = {
 449	"print-parent",
 450	"sym-offset",
 451	"sym-addr",
 452	"verbose",
 453	"raw",
 454	"hex",
 455	"bin",
 456	"block",
 457	"stacktrace",
 458	"trace_printk",
 459	"ftrace_preempt",
 460	"branch",
 461	"annotate",
 462	"userstacktrace",
 463	"sym-userobj",
 464	"printk-msg-only",
 465	"context-info",
 466	"latency-format",
 467	"sleep-time",
 468	"graph-time",
 469	"record-cmd",
 470	"overwrite",
 471	"disable_on_free",
 472	"irq-info",
 473	NULL
 474};
 475
 476static struct {
 477	u64 (*func)(void);
 478	const char *name;
 
 479} trace_clocks[] = {
 480	{ trace_clock_local,	"local" },
 481	{ trace_clock_global,	"global" },
 482	{ trace_clock_counter,	"counter" },
 
 
 
 
 
 
 
 483};
 484
 485int trace_clock_id;
 
 
 
 
 
 
 486
 487/*
 488 * trace_parser_get_init - gets the buffer for trace parser
 489 */
 490int trace_parser_get_init(struct trace_parser *parser, int size)
 491{
 492	memset(parser, 0, sizeof(*parser));
 493
 494	parser->buffer = kmalloc(size, GFP_KERNEL);
 495	if (!parser->buffer)
 496		return 1;
 497
 498	parser->size = size;
 499	return 0;
 500}
 501
 502/*
 503 * trace_parser_put - frees the buffer for trace parser
 504 */
 505void trace_parser_put(struct trace_parser *parser)
 506{
 507	kfree(parser->buffer);
 
 508}
 509
 510/*
 511 * trace_get_user - reads the user input string separated by  space
 512 * (matched by isspace(ch))
 513 *
 514 * For each string found the 'struct trace_parser' is updated,
 515 * and the function returns.
 516 *
 517 * Returns number of bytes read.
 518 *
 519 * See kernel/trace/trace.h for 'struct trace_parser' details.
 520 */
 521int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
 522	size_t cnt, loff_t *ppos)
 523{
 524	char ch;
 525	size_t read = 0;
 526	ssize_t ret;
 527
 528	if (!*ppos)
 529		trace_parser_clear(parser);
 530
 531	ret = get_user(ch, ubuf++);
 532	if (ret)
 533		goto out;
 534
 535	read++;
 536	cnt--;
 537
 538	/*
 539	 * The parser is not finished with the last write,
 540	 * continue reading the user input without skipping spaces.
 541	 */
 542	if (!parser->cont) {
 543		/* skip white space */
 544		while (cnt && isspace(ch)) {
 545			ret = get_user(ch, ubuf++);
 546			if (ret)
 547				goto out;
 548			read++;
 549			cnt--;
 550		}
 551
 
 
 552		/* only spaces were written */
 553		if (isspace(ch)) {
 554			*ppos += read;
 555			ret = read;
 556			goto out;
 557		}
 558
 559		parser->idx = 0;
 560	}
 561
 562	/* read the non-space input */
 563	while (cnt && !isspace(ch)) {
 564		if (parser->idx < parser->size - 1)
 565			parser->buffer[parser->idx++] = ch;
 566		else {
 567			ret = -EINVAL;
 568			goto out;
 569		}
 570		ret = get_user(ch, ubuf++);
 571		if (ret)
 572			goto out;
 573		read++;
 574		cnt--;
 575	}
 576
 577	/* We either got finished input or we have to wait for another call. */
 578	if (isspace(ch)) {
 579		parser->buffer[parser->idx] = 0;
 580		parser->cont = false;
 581	} else {
 582		parser->cont = true;
 583		parser->buffer[parser->idx++] = ch;
 
 
 
 
 
 584	}
 585
 586	*ppos += read;
 587	ret = read;
 588
 589out:
 590	return ret;
 591}
 592
 593ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
 
 594{
 595	int len;
 596	int ret;
 597
 598	if (!cnt)
 599		return 0;
 600
 601	if (s->len <= s->readpos)
 602		return -EBUSY;
 603
 604	len = s->len - s->readpos;
 605	if (cnt > len)
 606		cnt = len;
 607	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
 608	if (ret == cnt)
 609		return -EFAULT;
 610
 611	cnt -= ret;
 612
 613	s->readpos += cnt;
 614	return cnt;
 615}
 616
 617static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
 
 
 
 
 
 
 
 
 
 618{
 619	int len;
 
 
 
 620
 621	if (s->len <= s->readpos)
 622		return -EBUSY;
 
 
 
 
 623
 624	len = s->len - s->readpos;
 625	if (cnt > len)
 626		cnt = len;
 627	memcpy(buf, s->buffer + s->readpos, cnt);
 
 
 
 
 
 
 628
 629	s->readpos += cnt;
 630	return cnt;
 
 
 
 
 
 
 
 631}
 632
 633/*
 634 * ftrace_max_lock is used to protect the swapping of buffers
 635 * when taking a max snapshot. The buffers themselves are
 636 * protected by per_cpu spinlocks. But the action of the swap
 637 * needs its own lock.
 638 *
 639 * This is defined as a arch_spinlock_t in order to help
 640 * with performance when lockdep debugging is enabled.
 641 *
 642 * It is also used in other places outside the update_max_tr
 643 * so it needs to be defined outside of the
 644 * CONFIG_TRACER_MAX_TRACE.
 645 */
 646static arch_spinlock_t ftrace_max_lock =
 647	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
 648
 649unsigned long __read_mostly	tracing_thresh;
 
 
 
 
 
 
 
 
 
 
 650
 651#ifdef CONFIG_TRACER_MAX_TRACE
 652unsigned long __read_mostly	tracing_max_latency;
 
 
 
 
 
 653
 654/*
 655 * Copy the new maximum trace into the separate maximum-trace
 656 * structure. (this way the maximum trace is permanently saved,
 657 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
 658 */
 659static void
 660__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 661{
 662	struct trace_array_cpu *data = tr->data[cpu];
 663	struct trace_array_cpu *max_data;
 
 
 664
 665	max_tr.cpu = cpu;
 666	max_tr.time_start = data->preempt_timestamp;
 667
 668	max_data = max_tr.data[cpu];
 669	max_data->saved_latency = tracing_max_latency;
 670	max_data->critical_start = data->critical_start;
 671	max_data->critical_end = data->critical_end;
 672
 673	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
 674	max_data->pid = tsk->pid;
 675	max_data->uid = task_uid(tsk);
 
 
 
 
 
 
 
 
 676	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
 677	max_data->policy = tsk->policy;
 678	max_data->rt_priority = tsk->rt_priority;
 679
 680	/* record this tasks comm */
 681	tracing_record_cmdline(tsk);
 
 682}
 683
 684/**
 685 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 686 * @tr: tracer
 687 * @tsk: the task with the latency
 688 * @cpu: The cpu that initiated the trace.
 
 689 *
 690 * Flip the buffers between the @tr and the max_tr and record information
 691 * about which task was the cause of this latency.
 692 */
 693void
 694update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 
 695{
 696	struct ring_buffer *buf = tr->buffer;
 697
 698	if (trace_stop_count)
 699		return;
 700
 701	WARN_ON_ONCE(!irqs_disabled());
 702	if (!current_trace->use_max_tr) {
 703		WARN_ON_ONCE(1);
 
 
 704		return;
 705	}
 706	arch_spin_lock(&ftrace_max_lock);
 707
 708	tr->buffer = max_tr.buffer;
 709	max_tr.buffer = buf;
 
 
 
 
 
 
 
 
 
 
 
 
 
 710
 711	__update_max_tr(tr, tsk, cpu);
 712	arch_spin_unlock(&ftrace_max_lock);
 
 
 
 
 713}
 714
 715/**
 716 * update_max_tr_single - only copy one trace over, and reset the rest
 717 * @tr - tracer
 718 * @tsk - task with the latency
 719 * @cpu - the cpu of the buffer to copy.
 720 *
 721 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
 722 */
 723void
 724update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 725{
 726	int ret;
 727
 728	if (trace_stop_count)
 729		return;
 730
 731	WARN_ON_ONCE(!irqs_disabled());
 732	if (!current_trace->use_max_tr) {
 733		WARN_ON_ONCE(1);
 
 734		return;
 735	}
 736
 737	arch_spin_lock(&ftrace_max_lock);
 738
 739	ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
 740
 741	if (ret == -EBUSY) {
 742		/*
 743		 * We failed to swap the buffer due to a commit taking
 744		 * place on this CPU. We fail to record, but we reset
 745		 * the max trace buffer (no one writes directly to it)
 746		 * and flag that it failed.
 
 747		 */
 748		trace_array_printk(&max_tr, _THIS_IP_,
 749			"Failed to swap buffers due to commit in progress\n");
 750	}
 751
 752	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
 753
 754	__update_max_tr(tr, tsk, cpu);
 755	arch_spin_unlock(&ftrace_max_lock);
 756}
 
 757#endif /* CONFIG_TRACER_MAX_TRACE */
 758
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 759/**
 760 * register_tracer - register a tracer with the ftrace system.
 761 * @type - the plugin for the tracer
 762 *
 763 * Register a new plugin tracer.
 764 */
 765int register_tracer(struct tracer *type)
 766{
 767	struct tracer *t;
 768	int ret = 0;
 769
 770	if (!type->name) {
 771		pr_info("Tracer must have a name\n");
 772		return -1;
 773	}
 774
 775	if (strlen(type->name) >= MAX_TRACER_SIZE) {
 776		pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
 777		return -1;
 778	}
 779
 780	mutex_lock(&trace_types_lock);
 
 
 
 
 781
 782	tracing_selftest_running = true;
 783
 784	for (t = trace_types; t; t = t->next) {
 785		if (strcmp(type->name, t->name) == 0) {
 786			/* already found */
 787			pr_info("Tracer %s already registered\n",
 788				type->name);
 789			ret = -1;
 790			goto out;
 791		}
 792	}
 793
 794	if (!type->set_flag)
 795		type->set_flag = &dummy_set_flag;
 796	if (!type->flags)
 797		type->flags = &dummy_tracer_flags;
 798	else
 799		if (!type->flags->opts)
 800			type->flags->opts = dummy_tracer_opt;
 801	if (!type->wait_pipe)
 802		type->wait_pipe = default_wait_pipe;
 803
 804
 805#ifdef CONFIG_FTRACE_STARTUP_TEST
 806	if (type->selftest && !tracing_selftest_disabled) {
 807		struct tracer *saved_tracer = current_trace;
 808		struct trace_array *tr = &global_trace;
 809
 810		/*
 811		 * Run a selftest on this tracer.
 812		 * Here we reset the trace buffer, and set the current
 813		 * tracer to be this tracer. The tracer can then run some
 814		 * internal tracing to verify that everything is in order.
 815		 * If we fail, we do not register this tracer.
 816		 */
 817		tracing_reset_online_cpus(tr);
 818
 819		current_trace = type;
 820
 821		/* If we expanded the buffers, make sure the max is expanded too */
 822		if (ring_buffer_expanded && type->use_max_tr)
 823			ring_buffer_resize(max_tr.buffer, trace_buf_size,
 824						RING_BUFFER_ALL_CPUS);
 825
 826		/* the test is responsible for initializing and enabling */
 827		pr_info("Testing tracer %s: ", type->name);
 828		ret = type->selftest(type, tr);
 829		/* the test is responsible for resetting too */
 830		current_trace = saved_tracer;
 831		if (ret) {
 832			printk(KERN_CONT "FAILED!\n");
 833			goto out;
 834		}
 835		/* Only reset on passing, to avoid touching corrupted buffers */
 836		tracing_reset_online_cpus(tr);
 
 
 
 837
 838		/* Shrink the max buffer again */
 839		if (ring_buffer_expanded && type->use_max_tr)
 840			ring_buffer_resize(max_tr.buffer, 1,
 841						RING_BUFFER_ALL_CPUS);
 842
 843		printk(KERN_CONT "PASSED\n");
 844	}
 845#endif
 846
 847	type->next = trace_types;
 848	trace_types = type;
 
 849
 850 out:
 851	tracing_selftest_running = false;
 852	mutex_unlock(&trace_types_lock);
 853
 854	if (ret || !default_bootup_tracer)
 855		goto out_unlock;
 856
 857	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
 858		goto out_unlock;
 859
 860	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
 861	/* Do we want this tracer to start on bootup? */
 862	tracing_set_tracer(type->name);
 863	default_bootup_tracer = NULL;
 
 
 
 864	/* disable other selftests, since this will break it. */
 865	tracing_selftest_disabled = 1;
 866#ifdef CONFIG_FTRACE_STARTUP_TEST
 867	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
 868	       type->name);
 869#endif
 870
 871 out_unlock:
 872	return ret;
 873}
 874
 875void unregister_tracer(struct tracer *type)
 876{
 877	struct tracer **t;
 878
 879	mutex_lock(&trace_types_lock);
 880	for (t = &trace_types; *t; t = &(*t)->next) {
 881		if (*t == type)
 882			goto found;
 883	}
 884	pr_info("Tracer %s not registered\n", type->name);
 885	goto out;
 886
 887 found:
 888	*t = (*t)->next;
 889
 890	if (type == current_trace && tracer_enabled) {
 891		tracer_enabled = 0;
 892		tracing_stop();
 893		if (current_trace->stop)
 894			current_trace->stop(&global_trace);
 895		current_trace = &nop_trace;
 896	}
 897out:
 898	mutex_unlock(&trace_types_lock);
 899}
 900
 901void tracing_reset(struct trace_array *tr, int cpu)
 902{
 903	struct ring_buffer *buffer = tr->buffer;
 904
 905	ring_buffer_record_disable(buffer);
 906
 907	/* Make sure all commits have finished */
 908	synchronize_sched();
 909	ring_buffer_reset_cpu(buffer, cpu);
 910
 911	ring_buffer_record_enable(buffer);
 912}
 913
 914void tracing_reset_online_cpus(struct trace_array *tr)
 915{
 916	struct ring_buffer *buffer = tr->buffer;
 917	int cpu;
 
 
 918
 919	ring_buffer_record_disable(buffer);
 920
 921	/* Make sure all commits have finished */
 922	synchronize_sched();
 923
 924	tr->time_start = ftrace_now(tr->cpu);
 925
 926	for_each_online_cpu(cpu)
 927		ring_buffer_reset_cpu(buffer, cpu);
 928
 929	ring_buffer_record_enable(buffer);
 930}
 931
 932void tracing_reset_current(int cpu)
 
 933{
 934	tracing_reset(&global_trace, cpu);
 935}
 936
 937void tracing_reset_current_online_cpus(void)
 938{
 939	tracing_reset_online_cpus(&global_trace);
 940}
 941
 942#define SAVED_CMDLINES 128
 943#define NO_CMDLINE_MAP UINT_MAX
 944static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 945static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
 946static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
 947static int cmdline_idx;
 948static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
 949
 950/* temporary disable recording */
 951static atomic_t trace_record_cmdline_disabled __read_mostly;
 952
 953static void trace_init_cmdlines(void)
 954{
 955	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
 956	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
 957	cmdline_idx = 0;
 958}
 959
 960int is_tracing_stopped(void)
 961{
 962	return trace_stop_count;
 963}
 964
 965/**
 966 * ftrace_off_permanent - disable all ftrace code permanently
 967 *
 968 * This should only be called when a serious anomally has
 969 * been detected.  This will turn off the function tracing,
 970 * ring buffers, and other tracing utilites. It takes no
 971 * locks and can be called from any context.
 972 */
 973void ftrace_off_permanent(void)
 974{
 975	tracing_disabled = 1;
 976	ftrace_stop();
 977	tracing_off_permanent();
 978}
 979
 980/**
 981 * tracing_start - quick start of the tracer
 982 *
 983 * If tracing is enabled but was stopped by tracing_stop,
 984 * this will start the tracer back up.
 985 */
 986void tracing_start(void)
 987{
 988	struct ring_buffer *buffer;
 989	unsigned long flags;
 990
 991	if (tracing_disabled)
 992		return;
 993
 994	raw_spin_lock_irqsave(&tracing_start_lock, flags);
 995	if (--trace_stop_count) {
 996		if (trace_stop_count < 0) {
 997			/* Someone screwed up their debugging */
 998			WARN_ON_ONCE(1);
 999			trace_stop_count = 0;
1000		}
1001		goto out;
1002	}
1003
1004	/* Prevent the buffers from switching */
1005	arch_spin_lock(&ftrace_max_lock);
1006
1007	buffer = global_trace.buffer;
1008	if (buffer)
1009		ring_buffer_record_enable(buffer);
1010
1011	buffer = max_tr.buffer;
 
1012	if (buffer)
1013		ring_buffer_record_enable(buffer);
 
1014
1015	arch_spin_unlock(&ftrace_max_lock);
1016
1017	ftrace_start();
1018 out:
1019	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1020}
1021
1022/**
1023 * tracing_stop - quick stop of the tracer
1024 *
1025 * Light weight way to stop tracing. Use in conjunction with
1026 * tracing_start.
1027 */
1028void tracing_stop(void)
 
 
 
 
 
 
1029{
1030	struct ring_buffer *buffer;
1031	unsigned long flags;
1032
1033	ftrace_stop();
1034	raw_spin_lock_irqsave(&tracing_start_lock, flags);
1035	if (trace_stop_count++)
1036		goto out;
1037
1038	/* Prevent the buffers from switching */
1039	arch_spin_lock(&ftrace_max_lock);
1040
1041	buffer = global_trace.buffer;
1042	if (buffer)
1043		ring_buffer_record_disable(buffer);
1044
1045	buffer = max_tr.buffer;
 
1046	if (buffer)
1047		ring_buffer_record_disable(buffer);
 
1048
1049	arch_spin_unlock(&ftrace_max_lock);
1050
1051 out:
1052	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1053}
1054
1055void trace_stop_cmdline_recording(void);
 
 
 
 
 
 
 
 
 
1056
1057static void trace_save_cmdline(struct task_struct *tsk)
 
 
 
 
 
1058{
1059	unsigned pid, idx;
 
 
 
1060
1061	if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1062		return;
 
 
 
 
 
 
1063
1064	/*
1065	 * It's not the end of the world if we don't get
1066	 * the lock, but we also don't want to spin
1067	 * nor do we want to disable interrupts,
1068	 * so if we miss here, then better luck next time.
1069	 */
1070	if (!arch_spin_trylock(&trace_cmdline_lock))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1071		return;
1072
1073	idx = map_pid_to_cmdline[tsk->pid];
1074	if (idx == NO_CMDLINE_MAP) {
1075		idx = (cmdline_idx + 1) % SAVED_CMDLINES;
 
 
 
 
 
1076
1077		/*
1078		 * Check whether the cmdline buffer at idx has a pid
1079		 * mapped. We are going to overwrite that entry so we
1080		 * need to clear the map_pid_to_cmdline. Otherwise we
1081		 * would read the new comm for the old pid.
1082		 */
1083		pid = map_cmdline_to_pid[idx];
1084		if (pid != NO_CMDLINE_MAP)
1085			map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1086
1087		map_cmdline_to_pid[idx] = tsk->pid;
1088		map_pid_to_cmdline[tsk->pid] = idx;
1089
1090		cmdline_idx = idx;
 
 
 
 
 
1091	}
 
1092
1093	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
 
 
 
 
 
1094
1095	arch_spin_unlock(&trace_cmdline_lock);
 
 
1096}
1097
1098void trace_find_cmdline(int pid, char comm[])
 
 
 
 
 
 
 
 
1099{
1100	unsigned map;
1101
1102	if (!pid) {
1103		strcpy(comm, "<idle>");
1104		return;
1105	}
1106
1107	if (WARN_ON_ONCE(pid < 0)) {
1108		strcpy(comm, "<XXX>");
1109		return;
1110	}
1111
1112	if (pid > PID_MAX_DEFAULT) {
1113		strcpy(comm, "<...>");
1114		return;
1115	}
1116
1117	preempt_disable();
1118	arch_spin_lock(&trace_cmdline_lock);
1119	map = map_pid_to_cmdline[pid];
1120	if (map != NO_CMDLINE_MAP)
1121		strcpy(comm, saved_cmdlines[map]);
1122	else
1123		strcpy(comm, "<...>");
1124
1125	arch_spin_unlock(&trace_cmdline_lock);
1126	preempt_enable();
1127}
1128
1129void tracing_record_cmdline(struct task_struct *tsk)
1130{
1131	if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1132	    !tracing_is_on())
1133		return;
1134
1135	trace_save_cmdline(tsk);
1136}
 
 
1137
1138void
1139tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1140			     int pc)
1141{
1142	struct task_struct *tsk = current;
 
 
 
1143
1144	entry->preempt_count		= pc & 0xff;
1145	entry->pid			= (tsk) ? tsk->pid : 0;
1146	entry->padding			= 0;
1147	entry->flags =
1148#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1149		(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1150#else
1151		TRACE_FLAG_IRQS_NOSUPPORT |
1152#endif
1153		((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1154		((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1155		(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1156}
1157EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
 
1158
1159struct ring_buffer_event *
1160trace_buffer_lock_reserve(struct ring_buffer *buffer,
1161			  int type,
1162			  unsigned long len,
1163			  unsigned long flags, int pc)
1164{
1165	struct ring_buffer_event *event;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1166
1167	event = ring_buffer_lock_reserve(buffer, len);
1168	if (event != NULL) {
1169		struct trace_entry *ent = ring_buffer_event_data(event);
1170
1171		tracing_generic_entry_update(ent, flags, pc);
1172		ent->type = type;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1173	}
1174
1175	return event;
 
 
 
 
 
 
 
 
 
 
 
 
 
1176}
 
1177
1178static inline void
1179__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1180			     struct ring_buffer_event *event,
1181			     unsigned long flags, int pc,
1182			     int wake)
1183{
1184	ring_buffer_unlock_commit(buffer, event);
 
 
 
 
1185
1186	ftrace_trace_stack(buffer, flags, 6, pc);
1187	ftrace_trace_userstack(buffer, flags, pc);
 
1188
1189	if (wake)
1190		trace_wake_up();
1191}
 
1192
1193void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1194				struct ring_buffer_event *event,
1195				unsigned long flags, int pc)
1196{
1197	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1198}
1199
1200struct ring_buffer_event *
1201trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1202				  int type, unsigned long len,
1203				  unsigned long flags, int pc)
1204{
1205	*current_rb = global_trace.buffer;
1206	return trace_buffer_lock_reserve(*current_rb,
1207					 type, len, flags, pc);
 
 
1208}
1209EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1210
1211void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1212					struct ring_buffer_event *event,
1213					unsigned long flags, int pc)
1214{
1215	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1216}
1217EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1218
1219void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1220				       struct ring_buffer_event *event,
1221				       unsigned long flags, int pc)
1222{
1223	__trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1224}
1225EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
 
 
 
 
 
 
 
 
 
1226
1227void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1228					    struct ring_buffer_event *event,
1229					    unsigned long flags, int pc,
1230					    struct pt_regs *regs)
 
1231{
1232	ring_buffer_unlock_commit(buffer, event);
1233
1234	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1235	ftrace_trace_userstack(buffer, flags, pc);
 
 
 
 
 
 
1236}
1237EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1238
1239void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1240					 struct ring_buffer_event *event)
 
 
 
 
1241{
1242	ring_buffer_discard_commit(buffer, event);
1243}
1244EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1245
1246void
1247trace_function(struct trace_array *tr,
1248	       unsigned long ip, unsigned long parent_ip, unsigned long flags,
1249	       int pc)
1250{
1251	struct ftrace_event_call *call = &event_function;
1252	struct ring_buffer *buffer = tr->buffer;
1253	struct ring_buffer_event *event;
1254	struct ftrace_entry *entry;
1255
1256	/* If we are reading the ring buffer, don't trace */
1257	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1258		return;
1259
1260	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1261					  flags, pc);
1262	if (!event)
1263		return;
1264	entry	= ring_buffer_event_data(event);
1265	entry->ip			= ip;
1266	entry->parent_ip		= parent_ip;
1267
1268	if (!filter_check_discard(call, entry, buffer, event))
1269		ring_buffer_unlock_commit(buffer, event);
1270}
1271
1272void
1273ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1274       unsigned long ip, unsigned long parent_ip, unsigned long flags,
1275       int pc)
1276{
1277	if (likely(!atomic_read(&data->disabled)))
1278		trace_function(tr, ip, parent_ip, flags, pc);
1279}
1280
1281#ifdef CONFIG_STACKTRACE
1282
1283#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
 
 
 
 
1284struct ftrace_stack {
1285	unsigned long		calls[FTRACE_STACK_MAX_ENTRIES];
 
 
 
 
 
1286};
1287
1288static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1289static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1290
1291static void __ftrace_trace_stack(struct ring_buffer *buffer,
1292				 unsigned long flags,
1293				 int skip, int pc, struct pt_regs *regs)
1294{
1295	struct ftrace_event_call *call = &event_kernel_stack;
1296	struct ring_buffer_event *event;
 
 
1297	struct stack_entry *entry;
1298	struct stack_trace trace;
1299	int use_stack;
1300	int size = FTRACE_STACK_ENTRIES;
1301
1302	trace.nr_entries	= 0;
1303	trace.skip		= skip;
1304
1305	/*
1306	 * Since events can happen in NMIs there's no safe way to
1307	 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1308	 * or NMI comes in, it will just have to use the default
1309	 * FTRACE_STACK_SIZE.
1310	 */
 
 
 
 
 
1311	preempt_disable_notrace();
1312
1313	use_stack = ++__get_cpu_var(ftrace_stack_reserve);
 
 
 
 
 
1314	/*
1315	 * We don't need any atomic variables, just a barrier.
1316	 * If an interrupt comes in, we don't care, because it would
1317	 * have exited and put the counter back to what we want.
1318	 * We just need a barrier to keep gcc from moving things
1319	 * around.
1320	 */
1321	barrier();
1322	if (use_stack == 1) {
1323		trace.entries		= &__get_cpu_var(ftrace_stack).calls[0];
1324		trace.max_entries	= FTRACE_STACK_MAX_ENTRIES;
1325
1326		if (regs)
1327			save_stack_trace_regs(regs, &trace);
1328		else
1329			save_stack_trace(&trace);
1330
1331		if (trace.nr_entries > size)
1332			size = trace.nr_entries;
1333	} else
1334		/* From now on, use_stack is a boolean */
1335		use_stack = 0;
1336
1337	size *= sizeof(unsigned long);
1338
1339	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1340					  sizeof(*entry) + size, flags, pc);
 
1341	if (!event)
1342		goto out;
1343	entry = ring_buffer_event_data(event);
1344
1345	memset(&entry->caller, 0, size);
 
 
1346
1347	if (use_stack)
1348		memcpy(&entry->caller, trace.entries,
1349		       trace.nr_entries * sizeof(unsigned long));
1350	else {
1351		trace.max_entries	= FTRACE_STACK_ENTRIES;
1352		trace.entries		= entry->caller;
1353		if (regs)
1354			save_stack_trace_regs(regs, &trace);
1355		else
1356			save_stack_trace(&trace);
1357	}
1358
1359	entry->size = trace.nr_entries;
1360
1361	if (!filter_check_discard(call, entry, buffer, event))
1362		ring_buffer_unlock_commit(buffer, event);
1363
1364 out:
1365	/* Again, don't let gcc optimize things here */
1366	barrier();
1367	__get_cpu_var(ftrace_stack_reserve)--;
1368	preempt_enable_notrace();
1369
1370}
1371
1372void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1373			     int skip, int pc, struct pt_regs *regs)
 
 
1374{
1375	if (!(trace_flags & TRACE_ITER_STACKTRACE))
1376		return;
1377
1378	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
1379}
1380
1381void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1382			int skip, int pc)
1383{
1384	if (!(trace_flags & TRACE_ITER_STACKTRACE))
 
 
 
1385		return;
 
1386
1387	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1388}
1389
1390void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1391		   int pc)
1392{
1393	__ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
 
 
 
 
 
 
 
 
1394}
1395
1396/**
1397 * trace_dump_stack - record a stack back trace in the trace buffer
 
1398 */
1399void trace_dump_stack(void)
1400{
1401	unsigned long flags;
1402
1403	if (tracing_disabled || tracing_selftest_running)
1404		return;
1405
1406	local_save_flags(flags);
1407
1408	/* skipping 3 traces, seems to get us at the caller of this function */
1409	__ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
 
 
1410}
 
1411
 
1412static DEFINE_PER_CPU(int, user_stack_count);
1413
1414void
1415ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
 
1416{
1417	struct ftrace_event_call *call = &event_user_stack;
1418	struct ring_buffer_event *event;
1419	struct userstack_entry *entry;
1420	struct stack_trace trace;
1421
1422	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1423		return;
1424
1425	/*
1426	 * NMIs can not handle page faults, even with fix ups.
1427	 * The save user stack can (and often does) fault.
1428	 */
1429	if (unlikely(in_nmi()))
1430		return;
1431
1432	/*
1433	 * prevent recursion, since the user stack tracing may
1434	 * trigger other kernel events.
1435	 */
1436	preempt_disable();
1437	if (__this_cpu_read(user_stack_count))
1438		goto out;
1439
1440	__this_cpu_inc(user_stack_count);
1441
1442	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1443					  sizeof(*entry), flags, pc);
1444	if (!event)
1445		goto out_drop_count;
1446	entry	= ring_buffer_event_data(event);
1447
1448	entry->tgid		= current->tgid;
1449	memset(&entry->caller, 0, sizeof(entry->caller));
1450
1451	trace.nr_entries	= 0;
1452	trace.max_entries	= FTRACE_STACK_ENTRIES;
1453	trace.skip		= 0;
1454	trace.entries		= entry->caller;
1455
1456	save_stack_trace_user(&trace);
1457	if (!filter_check_discard(call, entry, buffer, event))
1458		ring_buffer_unlock_commit(buffer, event);
1459
1460 out_drop_count:
1461	__this_cpu_dec(user_stack_count);
1462 out:
1463	preempt_enable();
1464}
1465
1466#ifdef UNUSED
1467static void __trace_userstack(struct trace_array *tr, unsigned long flags)
 
1468{
1469	ftrace_trace_userstack(tr, flags, preempt_count());
1470}
1471#endif /* UNUSED */
1472
1473#endif /* CONFIG_STACKTRACE */
1474
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1475/* created for use with alloc_percpu */
1476struct trace_buffer_struct {
1477	char buffer[TRACE_BUF_SIZE];
 
1478};
1479
1480static struct trace_buffer_struct *trace_percpu_buffer;
1481static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1482static struct trace_buffer_struct *trace_percpu_irq_buffer;
1483static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1484
1485/*
1486 * The buffer used is dependent on the context. There is a per cpu
1487 * buffer for normal context, softirq contex, hard irq context and
1488 * for NMI context. Thise allows for lockless recording.
1489 *
1490 * Note, if the buffers failed to be allocated, then this returns NULL
1491 */
1492static char *get_trace_buf(void)
1493{
1494	struct trace_buffer_struct *percpu_buffer;
1495	struct trace_buffer_struct *buffer;
1496
1497	/*
1498	 * If we have allocated per cpu buffers, then we do not
1499	 * need to do any locking.
1500	 */
1501	if (in_nmi())
1502		percpu_buffer = trace_percpu_nmi_buffer;
1503	else if (in_irq())
1504		percpu_buffer = trace_percpu_irq_buffer;
1505	else if (in_softirq())
1506		percpu_buffer = trace_percpu_sirq_buffer;
1507	else
1508		percpu_buffer = trace_percpu_buffer;
1509
1510	if (!percpu_buffer)
1511		return NULL;
1512
1513	buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1514
1515	return buffer->buffer;
 
 
 
 
 
 
 
 
 
1516}
1517
1518static int alloc_percpu_trace_buffer(void)
1519{
1520	struct trace_buffer_struct *buffers;
1521	struct trace_buffer_struct *sirq_buffers;
1522	struct trace_buffer_struct *irq_buffers;
1523	struct trace_buffer_struct *nmi_buffers;
1524
1525	buffers = alloc_percpu(struct trace_buffer_struct);
1526	if (!buffers)
1527		goto err_warn;
1528
1529	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1530	if (!sirq_buffers)
1531		goto err_sirq;
1532
1533	irq_buffers = alloc_percpu(struct trace_buffer_struct);
1534	if (!irq_buffers)
1535		goto err_irq;
1536
1537	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1538	if (!nmi_buffers)
1539		goto err_nmi;
1540
1541	trace_percpu_buffer = buffers;
1542	trace_percpu_sirq_buffer = sirq_buffers;
1543	trace_percpu_irq_buffer = irq_buffers;
1544	trace_percpu_nmi_buffer = nmi_buffers;
1545
1546	return 0;
1547
1548 err_nmi:
1549	free_percpu(irq_buffers);
1550 err_irq:
1551	free_percpu(sirq_buffers);
1552 err_sirq:
1553	free_percpu(buffers);
1554 err_warn:
1555	WARN(1, "Could not allocate percpu trace_printk buffer");
1556	return -ENOMEM;
1557}
1558
 
 
1559void trace_printk_init_buffers(void)
1560{
1561	static int buffers_allocated;
1562
1563	if (buffers_allocated)
1564		return;
1565
1566	if (alloc_percpu_trace_buffer())
1567		return;
1568
1569	pr_info("ftrace: Allocated trace_printk buffers\n");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1570
1571	buffers_allocated = 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1572}
1573
1574/**
1575 * trace_vbprintk - write binary msg to tracing buffer
1576 *
 
 
1577 */
1578int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1579{
1580	struct ftrace_event_call *call = &event_bprint;
1581	struct ring_buffer_event *event;
1582	struct ring_buffer *buffer;
1583	struct trace_array *tr = &global_trace;
1584	struct bprint_entry *entry;
1585	unsigned long flags;
1586	char *tbuffer;
1587	int len = 0, size, pc;
1588
1589	if (unlikely(tracing_selftest_running || tracing_disabled))
1590		return 0;
1591
1592	/* Don't pollute graph traces with trace_vprintk internals */
1593	pause_graph_tracing();
1594
1595	pc = preempt_count();
1596	preempt_disable_notrace();
1597
1598	tbuffer = get_trace_buf();
1599	if (!tbuffer) {
1600		len = 0;
1601		goto out;
1602	}
1603
1604	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1605
1606	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1607		goto out;
1608
1609	local_save_flags(flags);
1610	size = sizeof(*entry) + sizeof(u32) * len;
1611	buffer = tr->buffer;
1612	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1613					  flags, pc);
 
1614	if (!event)
1615		goto out;
1616	entry = ring_buffer_event_data(event);
1617	entry->ip			= ip;
1618	entry->fmt			= fmt;
1619
1620	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1621	if (!filter_check_discard(call, entry, buffer, event)) {
1622		ring_buffer_unlock_commit(buffer, event);
1623		ftrace_trace_stack(buffer, flags, 6, pc);
1624	}
1625
1626out:
 
 
 
 
 
1627	preempt_enable_notrace();
1628	unpause_graph_tracing();
1629
1630	return len;
1631}
1632EXPORT_SYMBOL_GPL(trace_vbprintk);
1633
1634int trace_array_printk(struct trace_array *tr,
1635		       unsigned long ip, const char *fmt, ...)
1636{
1637	int ret;
1638	va_list ap;
1639
1640	if (!(trace_flags & TRACE_ITER_PRINTK))
1641		return 0;
1642
1643	va_start(ap, fmt);
1644	ret = trace_array_vprintk(tr, ip, fmt, ap);
1645	va_end(ap);
1646	return ret;
1647}
1648
1649int trace_array_vprintk(struct trace_array *tr,
1650			unsigned long ip, const char *fmt, va_list args)
1651{
1652	struct ftrace_event_call *call = &event_print;
1653	struct ring_buffer_event *event;
1654	struct ring_buffer *buffer;
1655	int len = 0, size, pc;
1656	struct print_entry *entry;
1657	unsigned long flags;
1658	char *tbuffer;
1659
1660	if (tracing_disabled || tracing_selftest_running)
1661		return 0;
1662
1663	/* Don't pollute graph traces with trace_vprintk internals */
1664	pause_graph_tracing();
1665
1666	pc = preempt_count();
1667	preempt_disable_notrace();
1668
1669
1670	tbuffer = get_trace_buf();
1671	if (!tbuffer) {
1672		len = 0;
1673		goto out;
1674	}
1675
1676	len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1677	if (len > TRACE_BUF_SIZE)
1678		goto out;
1679
1680	local_save_flags(flags);
1681	size = sizeof(*entry) + len + 1;
1682	buffer = tr->buffer;
1683	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1684					  flags, pc);
1685	if (!event)
1686		goto out;
1687	entry = ring_buffer_event_data(event);
1688	entry->ip = ip;
1689
1690	memcpy(&entry->buf, tbuffer, len);
1691	entry->buf[len] = '\0';
1692	if (!filter_check_discard(call, entry, buffer, event)) {
1693		ring_buffer_unlock_commit(buffer, event);
1694		ftrace_trace_stack(buffer, flags, 6, pc);
1695	}
1696 out:
 
 
 
 
 
1697	preempt_enable_notrace();
1698	unpause_graph_tracing();
1699
1700	return len;
1701}
1702
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1703int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1704{
1705	return trace_array_vprintk(&global_trace, ip, fmt, args);
1706}
1707EXPORT_SYMBOL_GPL(trace_vprintk);
1708
1709static void trace_iterator_increment(struct trace_iterator *iter)
1710{
 
 
1711	iter->idx++;
1712	if (iter->buffer_iter[iter->cpu])
1713		ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1714}
1715
1716static struct trace_entry *
1717peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1718		unsigned long *lost_events)
1719{
1720	struct ring_buffer_event *event;
1721	struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1722
1723	if (buf_iter)
1724		event = ring_buffer_iter_peek(buf_iter, ts);
1725	else
1726		event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
 
 
 
1727					 lost_events);
 
1728
1729	if (event) {
1730		iter->ent_size = ring_buffer_event_length(event);
1731		return ring_buffer_event_data(event);
1732	}
1733	iter->ent_size = 0;
1734	return NULL;
1735}
1736
1737static struct trace_entry *
1738__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1739		  unsigned long *missing_events, u64 *ent_ts)
1740{
1741	struct ring_buffer *buffer = iter->tr->buffer;
1742	struct trace_entry *ent, *next = NULL;
1743	unsigned long lost_events = 0, next_lost = 0;
1744	int cpu_file = iter->cpu_file;
1745	u64 next_ts = 0, ts;
1746	int next_cpu = -1;
1747	int next_size = 0;
1748	int cpu;
1749
1750	/*
1751	 * If we are in a per_cpu trace file, don't bother by iterating over
1752	 * all cpu and peek directly.
1753	 */
1754	if (cpu_file > TRACE_PIPE_ALL_CPU) {
1755		if (ring_buffer_empty_cpu(buffer, cpu_file))
1756			return NULL;
1757		ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1758		if (ent_cpu)
1759			*ent_cpu = cpu_file;
1760
1761		return ent;
1762	}
1763
1764	for_each_tracing_cpu(cpu) {
1765
1766		if (ring_buffer_empty_cpu(buffer, cpu))
1767			continue;
1768
1769		ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1770
1771		/*
1772		 * Pick the entry with the smallest timestamp:
1773		 */
1774		if (ent && (!next || ts < next_ts)) {
1775			next = ent;
1776			next_cpu = cpu;
1777			next_ts = ts;
1778			next_lost = lost_events;
1779			next_size = iter->ent_size;
1780		}
1781	}
1782
1783	iter->ent_size = next_size;
1784
1785	if (ent_cpu)
1786		*ent_cpu = next_cpu;
1787
1788	if (ent_ts)
1789		*ent_ts = next_ts;
1790
1791	if (missing_events)
1792		*missing_events = next_lost;
1793
1794	return next;
1795}
1796
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1797/* Find the next real entry, without updating the iterator itself */
1798struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1799					  int *ent_cpu, u64 *ent_ts)
1800{
1801	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1802}
1803
1804/* Find the next real entry, and increment the iterator to the next entry */
1805void *trace_find_next_entry_inc(struct trace_iterator *iter)
1806{
1807	iter->ent = __find_next_entry(iter, &iter->cpu,
1808				      &iter->lost_events, &iter->ts);
1809
1810	if (iter->ent)
1811		trace_iterator_increment(iter);
1812
1813	return iter->ent ? iter : NULL;
1814}
1815
1816static void trace_consume(struct trace_iterator *iter)
1817{
1818	ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1819			    &iter->lost_events);
1820}
1821
1822static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1823{
1824	struct trace_iterator *iter = m->private;
1825	int i = (int)*pos;
1826	void *ent;
1827
1828	WARN_ON_ONCE(iter->leftover);
1829
1830	(*pos)++;
1831
1832	/* can't go backwards */
1833	if (iter->idx > i)
1834		return NULL;
1835
1836	if (iter->idx < 0)
1837		ent = trace_find_next_entry_inc(iter);
1838	else
1839		ent = iter;
1840
1841	while (ent && iter->idx < i)
1842		ent = trace_find_next_entry_inc(iter);
1843
1844	iter->pos = *pos;
1845
1846	return ent;
1847}
1848
1849void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1850{
1851	struct trace_array *tr = iter->tr;
1852	struct ring_buffer_event *event;
1853	struct ring_buffer_iter *buf_iter;
1854	unsigned long entries = 0;
1855	u64 ts;
1856
1857	tr->data[cpu]->skipped_entries = 0;
1858
1859	if (!iter->buffer_iter[cpu])
 
1860		return;
1861
1862	buf_iter = iter->buffer_iter[cpu];
1863	ring_buffer_iter_reset(buf_iter);
1864
1865	/*
1866	 * We could have the case with the max latency tracers
1867	 * that a reset never took place on a cpu. This is evident
1868	 * by the timestamp being before the start of the buffer.
1869	 */
1870	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1871		if (ts >= iter->tr->time_start)
1872			break;
1873		entries++;
1874		ring_buffer_read(buf_iter, NULL);
1875	}
1876
1877	tr->data[cpu]->skipped_entries = entries;
1878}
1879
1880/*
1881 * The current tracer is copied to avoid a global locking
1882 * all around.
1883 */
1884static void *s_start(struct seq_file *m, loff_t *pos)
1885{
1886	struct trace_iterator *iter = m->private;
1887	static struct tracer *old_tracer;
1888	int cpu_file = iter->cpu_file;
1889	void *p = NULL;
1890	loff_t l = 0;
1891	int cpu;
1892
1893	/* copy the tracer to avoid using a global lock all around */
1894	mutex_lock(&trace_types_lock);
1895	if (unlikely(old_tracer != current_trace && current_trace)) {
1896		old_tracer = current_trace;
1897		*iter->trace = *current_trace;
 
 
 
 
 
1898	}
1899	mutex_unlock(&trace_types_lock);
1900
1901	atomic_inc(&trace_record_cmdline_disabled);
 
 
 
1902
1903	if (*pos != iter->pos) {
1904		iter->ent = NULL;
1905		iter->cpu = 0;
1906		iter->idx = -1;
1907
1908		if (cpu_file == TRACE_PIPE_ALL_CPU) {
1909			for_each_tracing_cpu(cpu)
1910				tracing_iter_reset(iter, cpu);
1911		} else
1912			tracing_iter_reset(iter, cpu_file);
1913
1914		iter->leftover = 0;
1915		for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1916			;
1917
1918	} else {
1919		/*
1920		 * If we overflowed the seq_file before, then we want
1921		 * to just reuse the trace_seq buffer again.
1922		 */
1923		if (iter->leftover)
1924			p = iter;
1925		else {
1926			l = *pos - 1;
1927			p = s_next(m, p, &l);
1928		}
1929	}
1930
1931	trace_event_read_lock();
1932	trace_access_lock(cpu_file);
1933	return p;
1934}
1935
1936static void s_stop(struct seq_file *m, void *p)
1937{
1938	struct trace_iterator *iter = m->private;
1939
1940	atomic_dec(&trace_record_cmdline_disabled);
 
 
 
 
1941	trace_access_unlock(iter->cpu_file);
1942	trace_event_read_unlock();
1943}
1944
1945static void
1946get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
 
1947{
1948	unsigned long count;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1949	int cpu;
1950
1951	*total = 0;
1952	*entries = 0;
1953
1954	for_each_tracing_cpu(cpu) {
1955		count = ring_buffer_entries_cpu(tr->buffer, cpu);
1956		/*
1957		 * If this buffer has skipped entries, then we hold all
1958		 * entries for the trace and we need to ignore the
1959		 * ones before the time stamp.
1960		 */
1961		if (tr->data[cpu]->skipped_entries) {
1962			count -= tr->data[cpu]->skipped_entries;
1963			/* total is the same as the entries */
1964			*total += count;
1965		} else
1966			*total += count +
1967				ring_buffer_overrun_cpu(tr->buffer, cpu);
1968		*entries += count;
1969	}
1970}
1971
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1972static void print_lat_help_header(struct seq_file *m)
1973{
1974	seq_puts(m, "#                  _------=> CPU#            \n");
1975	seq_puts(m, "#                 / _-----=> irqs-off        \n");
1976	seq_puts(m, "#                | / _----=> need-resched    \n");
1977	seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1978	seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1979	seq_puts(m, "#                |||| /     delay             \n");
1980	seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1981	seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
 
1982}
1983
1984static void print_event_info(struct trace_array *tr, struct seq_file *m)
1985{
1986	unsigned long total;
1987	unsigned long entries;
1988
1989	get_total_entries(tr, &total, &entries);
1990	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
1991		   entries, total, num_online_cpus());
1992	seq_puts(m, "#\n");
1993}
1994
1995static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
 
1996{
1997	print_event_info(tr, m);
1998	seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
1999	seq_puts(m, "#              | |       |          |         |\n");
2000}
2001
2002static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
2003{
2004	print_event_info(tr, m);
2005	seq_puts(m, "#                              _-----=> irqs-off\n");
2006	seq_puts(m, "#                             / _----=> need-resched\n");
2007	seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2008	seq_puts(m, "#                            || / _--=> preempt-depth\n");
2009	seq_puts(m, "#                            ||| /     delay\n");
2010	seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2011	seq_puts(m, "#              | |       |   ||||       |         |\n");
 
 
 
 
 
 
 
 
 
 
2012}
2013
2014void
2015print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2016{
2017	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2018	struct trace_array *tr = iter->tr;
2019	struct trace_array_cpu *data = tr->data[tr->cpu];
2020	struct tracer *type = current_trace;
2021	unsigned long entries;
2022	unsigned long total;
2023	const char *name = "preemption";
2024
2025	if (type)
2026		name = type->name;
2027
2028	get_total_entries(tr, &total, &entries);
2029
2030	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2031		   name, UTS_RELEASE);
2032	seq_puts(m, "# -----------------------------------"
2033		 "---------------------------------\n");
2034	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2035		   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2036		   nsecs_to_usecs(data->saved_latency),
2037		   entries,
2038		   total,
2039		   tr->cpu,
2040#if defined(CONFIG_PREEMPT_NONE)
2041		   "server",
2042#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2043		   "desktop",
2044#elif defined(CONFIG_PREEMPT)
2045		   "preempt",
2046#else
2047		   "unknown",
2048#endif
2049		   /* These are reserved for later use */
2050		   0, 0, 0, 0);
2051#ifdef CONFIG_SMP
2052	seq_printf(m, " #P:%d)\n", num_online_cpus());
2053#else
2054	seq_puts(m, ")\n");
2055#endif
2056	seq_puts(m, "#    -----------------\n");
2057	seq_printf(m, "#    | task: %.16s-%d "
2058		   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2059		   data->comm, data->pid, data->uid, data->nice,
 
2060		   data->policy, data->rt_priority);
2061	seq_puts(m, "#    -----------------\n");
2062
2063	if (data->critical_start) {
2064		seq_puts(m, "#  => started at: ");
2065		seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2066		trace_print_seq(m, &iter->seq);
2067		seq_puts(m, "\n#  => ended at:   ");
2068		seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2069		trace_print_seq(m, &iter->seq);
2070		seq_puts(m, "\n#\n");
2071	}
2072
2073	seq_puts(m, "#\n");
2074}
2075
2076static void test_cpu_buff_start(struct trace_iterator *iter)
2077{
2078	struct trace_seq *s = &iter->seq;
 
2079
2080	if (!(trace_flags & TRACE_ITER_ANNOTATE))
2081		return;
2082
2083	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2084		return;
2085
2086	if (cpumask_test_cpu(iter->cpu, iter->started))
 
2087		return;
2088
2089	if (iter->tr->data[iter->cpu]->skipped_entries)
2090		return;
2091
2092	cpumask_set_cpu(iter->cpu, iter->started);
 
2093
2094	/* Don't print started cpu buffer for the first entry of the trace */
2095	if (iter->idx > 1)
2096		trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2097				iter->cpu);
2098}
2099
2100static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2101{
 
2102	struct trace_seq *s = &iter->seq;
2103	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2104	struct trace_entry *entry;
2105	struct trace_event *event;
2106
2107	entry = iter->ent;
2108
2109	test_cpu_buff_start(iter);
2110
2111	event = ftrace_find_event(entry->type);
2112
2113	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2114		if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2115			if (!trace_print_lat_context(iter))
2116				goto partial;
2117		} else {
2118			if (!trace_print_context(iter))
2119				goto partial;
2120		}
2121	}
2122
2123	if (event)
 
 
 
 
 
2124		return event->funcs->trace(iter, sym_flags, event);
 
2125
2126	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2127		goto partial;
2128
2129	return TRACE_TYPE_HANDLED;
2130partial:
2131	return TRACE_TYPE_PARTIAL_LINE;
2132}
2133
2134static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2135{
 
2136	struct trace_seq *s = &iter->seq;
2137	struct trace_entry *entry;
2138	struct trace_event *event;
2139
2140	entry = iter->ent;
2141
2142	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2143		if (!trace_seq_printf(s, "%d %d %llu ",
2144				      entry->pid, iter->cpu, iter->ts))
2145			goto partial;
2146	}
 
2147
2148	event = ftrace_find_event(entry->type);
2149	if (event)
2150		return event->funcs->raw(iter, 0, event);
2151
2152	if (!trace_seq_printf(s, "%d ?\n", entry->type))
2153		goto partial;
2154
2155	return TRACE_TYPE_HANDLED;
2156partial:
2157	return TRACE_TYPE_PARTIAL_LINE;
2158}
2159
2160static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2161{
 
2162	struct trace_seq *s = &iter->seq;
2163	unsigned char newline = '\n';
2164	struct trace_entry *entry;
2165	struct trace_event *event;
2166
2167	entry = iter->ent;
2168
2169	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2170		SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2171		SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2172		SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
 
 
2173	}
2174
2175	event = ftrace_find_event(entry->type);
2176	if (event) {
2177		enum print_line_t ret = event->funcs->hex(iter, 0, event);
2178		if (ret != TRACE_TYPE_HANDLED)
2179			return ret;
2180	}
2181
2182	SEQ_PUT_FIELD_RET(s, newline);
2183
2184	return TRACE_TYPE_HANDLED;
2185}
2186
2187static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2188{
 
2189	struct trace_seq *s = &iter->seq;
2190	struct trace_entry *entry;
2191	struct trace_event *event;
2192
2193	entry = iter->ent;
2194
2195	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2196		SEQ_PUT_FIELD_RET(s, entry->pid);
2197		SEQ_PUT_FIELD_RET(s, iter->cpu);
2198		SEQ_PUT_FIELD_RET(s, iter->ts);
 
 
2199	}
2200
2201	event = ftrace_find_event(entry->type);
2202	return event ? event->funcs->binary(iter, 0, event) :
2203		TRACE_TYPE_HANDLED;
2204}
2205
2206int trace_empty(struct trace_iterator *iter)
2207{
 
2208	int cpu;
2209
2210	/* If we are looking at one CPU buffer, only check that one */
2211	if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2212		cpu = iter->cpu_file;
2213		if (iter->buffer_iter[cpu]) {
2214			if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
 
2215				return 0;
2216		} else {
2217			if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2218				return 0;
2219		}
2220		return 1;
2221	}
2222
2223	for_each_tracing_cpu(cpu) {
2224		if (iter->buffer_iter[cpu]) {
2225			if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
 
2226				return 0;
2227		} else {
2228			if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2229				return 0;
2230		}
2231	}
2232
2233	return 1;
2234}
2235
2236/*  Called with trace_event_read_lock() held. */
2237enum print_line_t print_trace_line(struct trace_iterator *iter)
2238{
 
 
2239	enum print_line_t ret;
2240
2241	if (iter->lost_events &&
2242	    !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2243				 iter->cpu, iter->lost_events))
2244		return TRACE_TYPE_PARTIAL_LINE;
 
 
 
 
 
 
2245
2246	if (iter->trace && iter->trace->print_line) {
2247		ret = iter->trace->print_line(iter);
2248		if (ret != TRACE_TYPE_UNHANDLED)
2249			return ret;
2250	}
2251
 
 
 
 
 
2252	if (iter->ent->type == TRACE_BPRINT &&
2253			trace_flags & TRACE_ITER_PRINTK &&
2254			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2255		return trace_print_bprintk_msg_only(iter);
2256
2257	if (iter->ent->type == TRACE_PRINT &&
2258			trace_flags & TRACE_ITER_PRINTK &&
2259			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2260		return trace_print_printk_msg_only(iter);
2261
2262	if (trace_flags & TRACE_ITER_BIN)
2263		return print_bin_fmt(iter);
2264
2265	if (trace_flags & TRACE_ITER_HEX)
2266		return print_hex_fmt(iter);
2267
2268	if (trace_flags & TRACE_ITER_RAW)
2269		return print_raw_fmt(iter);
2270
2271	return print_trace_fmt(iter);
2272}
2273
2274void trace_latency_header(struct seq_file *m)
2275{
2276	struct trace_iterator *iter = m->private;
 
2277
2278	/* print nothing if the buffers are empty */
2279	if (trace_empty(iter))
2280		return;
2281
2282	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2283		print_trace_header(m, iter);
2284
2285	if (!(trace_flags & TRACE_ITER_VERBOSE))
2286		print_lat_help_header(m);
2287}
2288
2289void trace_default_header(struct seq_file *m)
2290{
2291	struct trace_iterator *iter = m->private;
 
 
2292
2293	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2294		return;
2295
2296	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2297		/* print nothing if the buffers are empty */
2298		if (trace_empty(iter))
2299			return;
2300		print_trace_header(m, iter);
2301		if (!(trace_flags & TRACE_ITER_VERBOSE))
2302			print_lat_help_header(m);
2303	} else {
2304		if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2305			if (trace_flags & TRACE_ITER_IRQ_INFO)
2306				print_func_help_header_irq(iter->tr, m);
 
2307			else
2308				print_func_help_header(iter->tr, m);
 
2309		}
2310	}
2311}
2312
2313static void test_ftrace_alive(struct seq_file *m)
2314{
2315	if (!ftrace_is_dead())
2316		return;
2317	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2318	seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2319}
2320
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2321static int s_show(struct seq_file *m, void *v)
2322{
2323	struct trace_iterator *iter = v;
2324	int ret;
2325
2326	if (iter->ent == NULL) {
2327		if (iter->tr) {
2328			seq_printf(m, "# tracer: %s\n", iter->trace->name);
2329			seq_puts(m, "#\n");
2330			test_ftrace_alive(m);
2331		}
2332		if (iter->trace && iter->trace->print_header)
 
 
2333			iter->trace->print_header(m);
2334		else
2335			trace_default_header(m);
2336
2337	} else if (iter->leftover) {
2338		/*
2339		 * If we filled the seq_file buffer earlier, we
2340		 * want to just show it now.
2341		 */
2342		ret = trace_print_seq(m, &iter->seq);
2343
2344		/* ret should this time be zero, but you never know */
2345		iter->leftover = ret;
2346
2347	} else {
2348		print_trace_line(iter);
 
 
 
 
2349		ret = trace_print_seq(m, &iter->seq);
2350		/*
2351		 * If we overflow the seq_file buffer, then it will
2352		 * ask us for this data again at start up.
2353		 * Use that instead.
2354		 *  ret is 0 if seq_file write succeeded.
2355		 *        -1 otherwise.
2356		 */
2357		iter->leftover = ret;
2358	}
2359
2360	return 0;
2361}
2362
 
 
 
 
 
 
 
 
 
 
 
2363static const struct seq_operations tracer_seq_ops = {
2364	.start		= s_start,
2365	.next		= s_next,
2366	.stop		= s_stop,
2367	.show		= s_show,
2368};
2369
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2370static struct trace_iterator *
2371__tracing_open(struct inode *inode, struct file *file)
2372{
2373	long cpu_file = (long) inode->i_private;
2374	struct trace_iterator *iter;
2375	int cpu;
2376
2377	if (tracing_disabled)
2378		return ERR_PTR(-ENODEV);
2379
2380	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2381	if (!iter)
2382		return ERR_PTR(-ENOMEM);
2383
 
 
 
 
 
2384	/*
2385	 * We make a copy of the current tracer to avoid concurrent
2386	 * changes on it while we are reading.
 
 
 
 
2387	 */
2388	mutex_lock(&trace_types_lock);
2389	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2390	if (!iter->trace)
2391		goto fail;
2392
2393	if (current_trace)
2394		*iter->trace = *current_trace;
 
 
 
 
 
 
 
 
 
 
2395
2396	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2397		goto fail;
2398
2399	if (current_trace && current_trace->print_max)
2400		iter->tr = &max_tr;
 
 
 
 
2401	else
2402		iter->tr = &global_trace;
 
 
2403	iter->pos = -1;
 
2404	mutex_init(&iter->mutex);
2405	iter->cpu_file = cpu_file;
2406
2407	/* Notify the tracer early; before we stop tracing. */
2408	if (iter->trace && iter->trace->open)
2409		iter->trace->open(iter);
2410
2411	/* Annotate start of buffers if we had overruns */
2412	if (ring_buffer_overruns(iter->tr->buffer))
2413		iter->iter_flags |= TRACE_FILE_ANNOTATE;
2414
2415	/* stop the trace while dumping */
2416	tracing_stop();
 
 
 
 
 
 
 
 
2417
2418	if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2419		for_each_tracing_cpu(cpu) {
2420			iter->buffer_iter[cpu] =
2421				ring_buffer_read_prepare(iter->tr->buffer, cpu);
 
2422		}
2423		ring_buffer_read_prepare_sync();
2424		for_each_tracing_cpu(cpu) {
2425			ring_buffer_read_start(iter->buffer_iter[cpu]);
2426			tracing_iter_reset(iter, cpu);
2427		}
2428	} else {
2429		cpu = iter->cpu_file;
2430		iter->buffer_iter[cpu] =
2431			ring_buffer_read_prepare(iter->tr->buffer, cpu);
 
2432		ring_buffer_read_prepare_sync();
2433		ring_buffer_read_start(iter->buffer_iter[cpu]);
2434		tracing_iter_reset(iter, cpu);
2435	}
2436
2437	mutex_unlock(&trace_types_lock);
2438
2439	return iter;
2440
2441 fail:
2442	mutex_unlock(&trace_types_lock);
2443	kfree(iter->trace);
 
2444	seq_release_private(inode, file);
2445	return ERR_PTR(-ENOMEM);
2446}
2447
2448int tracing_open_generic(struct inode *inode, struct file *filp)
2449{
2450	if (tracing_disabled)
2451		return -ENODEV;
 
 
 
2452
2453	filp->private_data = inode->i_private;
2454	return 0;
2455}
2456
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2457static int tracing_release(struct inode *inode, struct file *file)
2458{
 
2459	struct seq_file *m = file->private_data;
2460	struct trace_iterator *iter;
2461	int cpu;
2462
2463	if (!(file->f_mode & FMODE_READ))
 
2464		return 0;
 
2465
 
2466	iter = m->private;
2467
2468	mutex_lock(&trace_types_lock);
 
2469	for_each_tracing_cpu(cpu) {
2470		if (iter->buffer_iter[cpu])
2471			ring_buffer_read_finish(iter->buffer_iter[cpu]);
2472	}
2473
2474	if (iter->trace && iter->trace->close)
2475		iter->trace->close(iter);
2476
2477	/* reenable tracing if it was previously enabled */
2478	tracing_start();
 
 
 
 
2479	mutex_unlock(&trace_types_lock);
2480
2481	mutex_destroy(&iter->mutex);
2482	free_cpumask_var(iter->started);
2483	kfree(iter->trace);
2484	seq_release_private(inode, file);
 
2485	return 0;
2486}
2487
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2488static int tracing_open(struct inode *inode, struct file *file)
2489{
 
2490	struct trace_iterator *iter;
2491	int ret = 0;
 
 
 
 
2492
2493	/* If this file was open for write, then erase contents */
2494	if ((file->f_mode & FMODE_WRITE) &&
2495	    (file->f_flags & O_TRUNC)) {
2496		long cpu = (long) inode->i_private;
2497
2498		if (cpu == TRACE_PIPE_ALL_CPU)
2499			tracing_reset_online_cpus(&global_trace);
 
 
 
 
 
2500		else
2501			tracing_reset(&global_trace, cpu);
2502	}
2503
2504	if (file->f_mode & FMODE_READ) {
2505		iter = __tracing_open(inode, file);
2506		if (IS_ERR(iter))
2507			ret = PTR_ERR(iter);
2508		else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2509			iter->iter_flags |= TRACE_FILE_LAT_FMT;
2510	}
 
 
 
 
2511	return ret;
2512}
2513
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2514static void *
2515t_next(struct seq_file *m, void *v, loff_t *pos)
2516{
 
2517	struct tracer *t = v;
2518
2519	(*pos)++;
2520
2521	if (t)
2522		t = t->next;
2523
2524	return t;
2525}
2526
2527static void *t_start(struct seq_file *m, loff_t *pos)
2528{
 
2529	struct tracer *t;
2530	loff_t l = 0;
2531
2532	mutex_lock(&trace_types_lock);
2533	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2534		;
 
 
2535
2536	return t;
2537}
2538
2539static void t_stop(struct seq_file *m, void *p)
2540{
2541	mutex_unlock(&trace_types_lock);
2542}
2543
2544static int t_show(struct seq_file *m, void *v)
2545{
2546	struct tracer *t = v;
2547
2548	if (!t)
2549		return 0;
2550
2551	seq_printf(m, "%s", t->name);
2552	if (t->next)
2553		seq_putc(m, ' ');
2554	else
2555		seq_putc(m, '\n');
2556
2557	return 0;
2558}
2559
2560static const struct seq_operations show_traces_seq_ops = {
2561	.start		= t_start,
2562	.next		= t_next,
2563	.stop		= t_stop,
2564	.show		= t_show,
2565};
2566
2567static int show_traces_open(struct inode *inode, struct file *file)
2568{
2569	if (tracing_disabled)
2570		return -ENODEV;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2571
2572	return seq_open(file, &show_traces_seq_ops);
 
2573}
2574
2575static ssize_t
2576tracing_write_stub(struct file *filp, const char __user *ubuf,
2577		   size_t count, loff_t *ppos)
2578{
2579	return count;
2580}
2581
2582static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2583{
 
 
2584	if (file->f_mode & FMODE_READ)
2585		return seq_lseek(file, offset, origin);
2586	else
2587		return 0;
 
 
2588}
2589
2590static const struct file_operations tracing_fops = {
2591	.open		= tracing_open,
2592	.read		= seq_read,
 
 
2593	.write		= tracing_write_stub,
2594	.llseek		= tracing_seek,
2595	.release	= tracing_release,
2596};
2597
2598static const struct file_operations show_traces_fops = {
2599	.open		= show_traces_open,
2600	.read		= seq_read,
2601	.release	= seq_release,
2602	.llseek		= seq_lseek,
 
2603};
2604
2605/*
2606 * Only trace on a CPU if the bitmask is set:
2607 */
2608static cpumask_var_t tracing_cpumask;
2609
2610/*
2611 * The tracer itself will not take this lock, but still we want
2612 * to provide a consistent cpumask to user-space:
2613 */
2614static DEFINE_MUTEX(tracing_cpumask_update_lock);
2615
2616/*
2617 * Temporary storage for the character representation of the
2618 * CPU bitmask (and one more byte for the newline):
2619 */
2620static char mask_str[NR_CPUS + 1];
2621
2622static ssize_t
2623tracing_cpumask_read(struct file *filp, char __user *ubuf,
2624		     size_t count, loff_t *ppos)
2625{
 
 
2626	int len;
2627
2628	mutex_lock(&tracing_cpumask_update_lock);
 
 
 
 
2629
2630	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2631	if (count - len < 2) {
 
2632		count = -EINVAL;
2633		goto out_err;
2634	}
2635	len += sprintf(mask_str + len, "\n");
2636	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2637
2638out_err:
2639	mutex_unlock(&tracing_cpumask_update_lock);
2640
2641	return count;
2642}
2643
2644static ssize_t
2645tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2646		      size_t count, loff_t *ppos)
2647{
2648	int err, cpu;
2649	cpumask_var_t tracing_cpumask_new;
2650
2651	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2652		return -ENOMEM;
2653
2654	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2655	if (err)
2656		goto err_unlock;
2657
2658	mutex_lock(&tracing_cpumask_update_lock);
 
2659
2660	local_irq_disable();
2661	arch_spin_lock(&ftrace_max_lock);
2662	for_each_tracing_cpu(cpu) {
2663		/*
2664		 * Increase/decrease the disabled counter if we are
2665		 * about to flip a bit in the cpumask:
2666		 */
2667		if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2668				!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2669			atomic_inc(&global_trace.data[cpu]->disabled);
2670			ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
 
 
 
2671		}
2672		if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2673				cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2674			atomic_dec(&global_trace.data[cpu]->disabled);
2675			ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
 
 
 
2676		}
2677	}
2678	arch_spin_unlock(&ftrace_max_lock);
2679	local_irq_enable();
2680
2681	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2682
2683	mutex_unlock(&tracing_cpumask_update_lock);
2684	free_cpumask_var(tracing_cpumask_new);
2685
2686	return count;
2687
2688err_unlock:
2689	free_cpumask_var(tracing_cpumask_new);
2690
2691	return err;
2692}
2693
2694static const struct file_operations tracing_cpumask_fops = {
2695	.open		= tracing_open_generic,
2696	.read		= tracing_cpumask_read,
2697	.write		= tracing_cpumask_write,
 
2698	.llseek		= generic_file_llseek,
2699};
2700
2701static int tracing_trace_options_show(struct seq_file *m, void *v)
2702{
2703	struct tracer_opt *trace_opts;
 
2704	u32 tracer_flags;
2705	int i;
2706
2707	mutex_lock(&trace_types_lock);
2708	tracer_flags = current_trace->flags->val;
2709	trace_opts = current_trace->flags->opts;
2710
2711	for (i = 0; trace_options[i]; i++) {
2712		if (trace_flags & (1 << i))
2713			seq_printf(m, "%s\n", trace_options[i]);
2714		else
2715			seq_printf(m, "no%s\n", trace_options[i]);
2716	}
2717
2718	for (i = 0; trace_opts[i].name; i++) {
2719		if (tracer_flags & trace_opts[i].bit)
2720			seq_printf(m, "%s\n", trace_opts[i].name);
2721		else
2722			seq_printf(m, "no%s\n", trace_opts[i].name);
2723	}
2724	mutex_unlock(&trace_types_lock);
2725
2726	return 0;
2727}
2728
2729static int __set_tracer_option(struct tracer *trace,
2730			       struct tracer_flags *tracer_flags,
2731			       struct tracer_opt *opts, int neg)
2732{
 
2733	int ret;
2734
2735	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2736	if (ret)
2737		return ret;
2738
2739	if (neg)
2740		tracer_flags->val &= ~opts->bit;
2741	else
2742		tracer_flags->val |= opts->bit;
2743	return 0;
2744}
2745
2746/* Try to assign a tracer specific option */
2747static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2748{
 
2749	struct tracer_flags *tracer_flags = trace->flags;
2750	struct tracer_opt *opts = NULL;
2751	int i;
2752
2753	for (i = 0; tracer_flags->opts[i].name; i++) {
2754		opts = &tracer_flags->opts[i];
2755
2756		if (strcmp(cmp, opts->name) == 0)
2757			return __set_tracer_option(trace, trace->flags,
2758						   opts, neg);
2759	}
2760
2761	return -EINVAL;
2762}
2763
2764static void set_tracer_flags(unsigned int mask, int enabled)
 
2765{
 
 
 
 
 
 
 
 
 
 
 
 
2766	/* do nothing if flag is already set */
2767	if (!!(trace_flags & mask) == !!enabled)
2768		return;
 
 
 
 
 
2769
2770	if (enabled)
2771		trace_flags |= mask;
2772	else
2773		trace_flags &= ~mask;
2774
2775	if (mask == TRACE_ITER_RECORD_CMD)
2776		trace_event_enable_cmd_record(enabled);
2777
2778	if (mask == TRACE_ITER_OVERWRITE)
2779		ring_buffer_change_overwrite(global_trace.buffer, enabled);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2780}
2781
2782static ssize_t
2783tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2784			size_t cnt, loff_t *ppos)
2785{
 
 
2786	char buf[64];
2787	char *cmp;
2788	int neg = 0;
2789	int ret;
2790	int i;
2791
2792	if (cnt >= sizeof(buf))
2793		return -EINVAL;
2794
2795	if (copy_from_user(&buf, ubuf, cnt))
2796		return -EFAULT;
2797
2798	buf[cnt] = 0;
2799	cmp = strstrip(buf);
2800
2801	if (strncmp(cmp, "no", 2) == 0) {
2802		neg = 1;
2803		cmp += 2;
2804	}
2805
2806	for (i = 0; trace_options[i]; i++) {
2807		if (strcmp(cmp, trace_options[i]) == 0) {
2808			set_tracer_flags(1 << i, !neg);
2809			break;
2810		}
2811	}
2812
2813	/* If no option could be set, test the specific tracer options */
2814	if (!trace_options[i]) {
2815		mutex_lock(&trace_types_lock);
2816		ret = set_tracer_option(current_trace, cmp, neg);
2817		mutex_unlock(&trace_types_lock);
2818		if (ret)
2819			return ret;
2820	}
2821
2822	*ppos += cnt;
2823
2824	return cnt;
2825}
2826
2827static int tracing_trace_options_open(struct inode *inode, struct file *file)
2828{
2829	if (tracing_disabled)
2830		return -ENODEV;
2831	return single_open(file, tracing_trace_options_show, NULL);
 
 
 
 
 
 
 
 
 
2832}
2833
2834static const struct file_operations tracing_iter_fops = {
2835	.open		= tracing_trace_options_open,
2836	.read		= seq_read,
2837	.llseek		= seq_lseek,
2838	.release	= single_release,
2839	.write		= tracing_trace_options_write,
2840};
2841
2842static const char readme_msg[] =
2843	"tracing mini-HOWTO:\n\n"
2844	"# mount -t debugfs nodev /sys/kernel/debug\n\n"
2845	"# cat /sys/kernel/debug/tracing/available_tracers\n"
2846	"wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2847	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2848	"nop\n"
2849	"# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2850	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2851	"wakeup\n"
2852	"# cat /sys/kernel/debug/tracing/trace_options\n"
2853	"noprint-parent nosym-offset nosym-addr noverbose\n"
2854	"# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2855	"# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2856	"# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2857	"# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2858;
2859
2860static ssize_t
2861tracing_readme_read(struct file *filp, char __user *ubuf,
2862		       size_t cnt, loff_t *ppos)
2863{
2864	return simple_read_from_buffer(ubuf, cnt, ppos,
2865					readme_msg, strlen(readme_msg));
2866}
2867
2868static const struct file_operations tracing_readme_fops = {
2869	.open		= tracing_open_generic,
2870	.read		= tracing_readme_read,
2871	.llseek		= generic_file_llseek,
2872};
2873
2874static ssize_t
2875tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2876				size_t cnt, loff_t *ppos)
2877{
2878	char *buf_comm;
2879	char *file_buf;
2880	char *buf;
2881	int len = 0;
2882	int pid;
2883	int i;
 
 
 
 
2884
2885	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2886	if (!file_buf)
2887		return -ENOMEM;
2888
2889	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2890	if (!buf_comm) {
2891		kfree(file_buf);
2892		return -ENOMEM;
2893	}
 
 
 
2894
2895	buf = file_buf;
 
2896
2897	for (i = 0; i < SAVED_CMDLINES; i++) {
2898		int r;
2899
2900		pid = map_cmdline_to_pid[i];
2901		if (pid == -1 || pid == NO_CMDLINE_MAP)
2902			continue;
 
2903
2904		trace_find_cmdline(pid, buf_comm);
2905		r = sprintf(buf, "%d %s\n", pid, buf_comm);
2906		buf += r;
2907		len += r;
2908	}
2909
2910	len = simple_read_from_buffer(ubuf, cnt, ppos,
2911				      file_buf, len);
 
2912
2913	kfree(file_buf);
2914	kfree(buf_comm);
 
2915
2916	return len;
2917}
2918
2919static const struct file_operations tracing_saved_cmdlines_fops = {
2920    .open       = tracing_open_generic,
2921    .read       = tracing_saved_cmdlines_read,
2922    .llseek	= generic_file_llseek,
2923};
2924
2925static ssize_t
2926tracing_ctrl_read(struct file *filp, char __user *ubuf,
2927		  size_t cnt, loff_t *ppos)
2928{
2929	char buf[64];
2930	int r;
2931
2932	r = sprintf(buf, "%u\n", tracer_enabled);
2933	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 
 
 
2934}
2935
2936static ssize_t
2937tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2938		   size_t cnt, loff_t *ppos)
 
 
 
 
 
2939{
2940	struct trace_array *tr = filp->private_data;
2941	unsigned long val;
2942	int ret;
2943
2944	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2945	if (ret)
2946		return ret;
2947
2948	val = !!val;
 
2949
2950	mutex_lock(&trace_types_lock);
2951	if (tracer_enabled ^ val) {
 
 
 
 
2952
2953		/* Only need to warn if this is used to change the state */
2954		WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2955
2956		if (val) {
2957			tracer_enabled = 1;
2958			if (current_trace->start)
2959				current_trace->start(tr);
2960			tracing_start();
2961		} else {
2962			tracer_enabled = 0;
2963			tracing_stop();
2964			if (current_trace->stop)
2965				current_trace->stop(tr);
2966		}
 
2967	}
2968	mutex_unlock(&trace_types_lock);
 
 
2969
2970	*ppos += cnt;
 
 
 
 
2971
2972	return cnt;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2973}
2974
2975static ssize_t
2976tracing_set_trace_read(struct file *filp, char __user *ubuf,
2977		       size_t cnt, loff_t *ppos)
2978{
 
2979	char buf[MAX_TRACER_SIZE+2];
2980	int r;
2981
2982	mutex_lock(&trace_types_lock);
2983	if (current_trace)
2984		r = sprintf(buf, "%s\n", current_trace->name);
2985	else
2986		r = sprintf(buf, "\n");
2987	mutex_unlock(&trace_types_lock);
2988
2989	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2990}
2991
2992int tracer_init(struct tracer *t, struct trace_array *tr)
2993{
2994	tracing_reset_online_cpus(tr);
2995	return t->init(tr);
2996}
2997
2998static void set_buffer_entries(struct trace_array *tr, unsigned long val)
2999{
3000	int cpu;
 
3001	for_each_tracing_cpu(cpu)
3002		tr->data[cpu]->entries = val;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3003}
 
3004
3005static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
 
3006{
3007	int ret;
3008
3009	/*
3010	 * If kernel or user changes the size of the ring buffer
3011	 * we use the size that was given, and we can forget about
3012	 * expanding it later.
3013	 */
3014	ring_buffer_expanded = 1;
3015
3016	ret = ring_buffer_resize(global_trace.buffer, size, cpu);
 
 
 
 
 
 
 
3017	if (ret < 0)
3018		return ret;
3019
3020	if (!current_trace->use_max_tr)
 
3021		goto out;
3022
3023	ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3024	if (ret < 0) {
3025		int r = 0;
3026
3027		if (cpu == RING_BUFFER_ALL_CPUS) {
3028			int i;
3029			for_each_tracing_cpu(i) {
3030				r = ring_buffer_resize(global_trace.buffer,
3031						global_trace.data[i]->entries,
3032						i);
3033				if (r < 0)
3034					break;
3035			}
3036		} else {
3037			r = ring_buffer_resize(global_trace.buffer,
3038						global_trace.data[cpu]->entries,
3039						cpu);
3040		}
3041
3042		if (r < 0) {
3043			/*
3044			 * AARGH! We are left with different
3045			 * size max buffer!!!!
3046			 * The max buffer is our "snapshot" buffer.
3047			 * When a tracer needs a snapshot (one of the
3048			 * latency tracers), it swaps the max buffer
3049			 * with the saved snap shot. We succeeded to
3050			 * update the size of the main buffer, but failed to
3051			 * update the size of the max buffer. But when we tried
3052			 * to reset the main buffer to the original size, we
3053			 * failed there too. This is very unlikely to
3054			 * happen, but if it does, warn and kill all
3055			 * tracing.
3056			 */
3057			WARN_ON(1);
3058			tracing_disabled = 1;
3059		}
3060		return ret;
3061	}
3062
3063	if (cpu == RING_BUFFER_ALL_CPUS)
3064		set_buffer_entries(&max_tr, size);
3065	else
3066		max_tr.data[cpu]->entries = size;
3067
3068 out:
3069	if (cpu == RING_BUFFER_ALL_CPUS)
3070		set_buffer_entries(&global_trace, size);
3071	else
3072		global_trace.data[cpu]->entries = size;
3073
 
 
 
3074	return ret;
3075}
3076
3077static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
 
3078{
3079	int ret = size;
3080
3081	mutex_lock(&trace_types_lock);
3082
3083	if (cpu_id != RING_BUFFER_ALL_CPUS) {
3084		/* make sure, this cpu is enabled in the mask */
3085		if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3086			ret = -EINVAL;
3087			goto out;
3088		}
3089	}
3090
3091	ret = __tracing_resize_ring_buffer(size, cpu_id);
3092	if (ret < 0)
3093		ret = -ENOMEM;
3094
3095out:
3096	mutex_unlock(&trace_types_lock);
3097
3098	return ret;
3099}
3100
3101
3102/**
3103 * tracing_update_buffers - used by tracing facility to expand ring buffers
 
3104 *
3105 * To save on memory when the tracing is never used on a system with it
3106 * configured in. The ring buffers are set to a minimum size. But once
3107 * a user starts to use the tracing facility, then they need to grow
3108 * to their default size.
3109 *
3110 * This function is to be called when a tracer is about to be used.
3111 */
3112int tracing_update_buffers(void)
3113{
3114	int ret = 0;
3115
3116	mutex_lock(&trace_types_lock);
3117	if (!ring_buffer_expanded)
3118		ret = __tracing_resize_ring_buffer(trace_buf_size,
3119						RING_BUFFER_ALL_CPUS);
3120	mutex_unlock(&trace_types_lock);
3121
3122	return ret;
3123}
3124
3125struct trace_option_dentry;
3126
3127static struct trace_option_dentry *
3128create_trace_option_files(struct tracer *tracer);
3129
3130static void
3131destroy_trace_option_files(struct trace_option_dentry *topts);
3132
3133static int tracing_set_tracer(const char *buf)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3134{
3135	static struct trace_option_dentry *topts;
3136	struct trace_array *tr = &global_trace;
3137	struct tracer *t;
 
 
 
3138	int ret = 0;
3139
3140	mutex_lock(&trace_types_lock);
3141
3142	if (!ring_buffer_expanded) {
3143		ret = __tracing_resize_ring_buffer(trace_buf_size,
3144						RING_BUFFER_ALL_CPUS);
3145		if (ret < 0)
3146			goto out;
3147		ret = 0;
3148	}
3149
3150	for (t = trace_types; t; t = t->next) {
3151		if (strcmp(t->name, buf) == 0)
3152			break;
3153	}
3154	if (!t) {
3155		ret = -EINVAL;
3156		goto out;
3157	}
3158	if (t == current_trace)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3159		goto out;
 
3160
3161	trace_branch_disable();
3162	if (current_trace && current_trace->reset)
3163		current_trace->reset(tr);
3164	if (current_trace && current_trace->use_max_tr) {
 
 
 
 
 
 
 
 
 
 
3165		/*
3166		 * We don't free the ring buffer. instead, resize it because
3167		 * The max_tr ring buffer has some state (e.g. ring->clock) and
3168		 * we want preserve it.
 
 
3169		 */
3170		ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3171		set_buffer_entries(&max_tr, 1);
 
3172	}
3173	destroy_trace_option_files(topts);
3174
3175	current_trace = t;
3176
3177	topts = create_trace_option_files(current_trace);
3178	if (current_trace->use_max_tr) {
3179		int cpu;
3180		/* we need to make per cpu buffer sizes equivalent */
3181		for_each_tracing_cpu(cpu) {
3182			ret = ring_buffer_resize(max_tr.buffer,
3183						global_trace.data[cpu]->entries,
3184						cpu);
3185			if (ret < 0)
3186				goto out;
3187			max_tr.data[cpu]->entries =
3188					global_trace.data[cpu]->entries;
3189		}
3190	}
 
 
 
3191
3192	if (t->init) {
3193		ret = tracer_init(t, tr);
3194		if (ret)
 
 
 
 
3195			goto out;
 
3196	}
3197
 
 
3198	trace_branch_enable(tr);
3199 out:
3200	mutex_unlock(&trace_types_lock);
3201
3202	return ret;
3203}
3204
3205static ssize_t
3206tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3207			size_t cnt, loff_t *ppos)
3208{
 
3209	char buf[MAX_TRACER_SIZE+1];
3210	int i;
3211	size_t ret;
3212	int err;
3213
3214	ret = cnt;
3215
3216	if (cnt > MAX_TRACER_SIZE)
3217		cnt = MAX_TRACER_SIZE;
3218
3219	if (copy_from_user(&buf, ubuf, cnt))
3220		return -EFAULT;
3221
3222	buf[cnt] = 0;
3223
3224	/* strip ending whitespace. */
3225	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3226		buf[i] = 0;
3227
3228	err = tracing_set_tracer(buf);
3229	if (err)
3230		return err;
3231
3232	*ppos += ret;
3233
3234	return ret;
3235}
3236
3237static ssize_t
3238tracing_max_lat_read(struct file *filp, char __user *ubuf,
3239		     size_t cnt, loff_t *ppos)
3240{
3241	unsigned long *ptr = filp->private_data;
3242	char buf[64];
3243	int r;
3244
3245	r = snprintf(buf, sizeof(buf), "%ld\n",
3246		     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3247	if (r > sizeof(buf))
3248		r = sizeof(buf);
3249	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3250}
3251
3252static ssize_t
3253tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3254		      size_t cnt, loff_t *ppos)
3255{
3256	unsigned long *ptr = filp->private_data;
3257	unsigned long val;
3258	int ret;
3259
3260	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3261	if (ret)
3262		return ret;
3263
3264	*ptr = val * 1000;
3265
3266	return cnt;
3267}
3268
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3269static int tracing_open_pipe(struct inode *inode, struct file *filp)
3270{
3271	long cpu_file = (long) inode->i_private;
3272	struct trace_iterator *iter;
3273	int ret = 0;
 
3274
3275	if (tracing_disabled)
3276		return -ENODEV;
 
3277
3278	mutex_lock(&trace_types_lock);
 
 
 
 
3279
3280	/* create a buffer to store the information to pass to userspace */
3281	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3282	if (!iter) {
3283		ret = -ENOMEM;
3284		goto out;
3285	}
3286
3287	/*
3288	 * We make a copy of the current tracer to avoid concurrent
3289	 * changes on it while we are reading.
3290	 */
3291	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3292	if (!iter->trace) {
3293		ret = -ENOMEM;
3294		goto fail;
3295	}
3296	if (current_trace)
3297		*iter->trace = *current_trace;
3298
3299	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3300		ret = -ENOMEM;
3301		goto fail;
3302	}
3303
3304	/* trace pipe does not show start of buffer */
3305	cpumask_setall(iter->started);
3306
3307	if (trace_flags & TRACE_ITER_LATENCY_FMT)
3308		iter->iter_flags |= TRACE_FILE_LAT_FMT;
3309
3310	iter->cpu_file = cpu_file;
3311	iter->tr = &global_trace;
 
 
 
 
 
3312	mutex_init(&iter->mutex);
3313	filp->private_data = iter;
3314
3315	if (iter->trace->pipe_open)
3316		iter->trace->pipe_open(iter);
3317
3318	nonseekable_open(inode, filp);
3319out:
 
 
3320	mutex_unlock(&trace_types_lock);
3321	return ret;
3322
3323fail:
3324	kfree(iter->trace);
3325	kfree(iter);
 
 
 
 
3326	mutex_unlock(&trace_types_lock);
3327	return ret;
3328}
3329
3330static int tracing_release_pipe(struct inode *inode, struct file *file)
3331{
3332	struct trace_iterator *iter = file->private_data;
 
3333
3334	mutex_lock(&trace_types_lock);
3335
 
 
3336	if (iter->trace->pipe_close)
3337		iter->trace->pipe_close(iter);
3338
3339	mutex_unlock(&trace_types_lock);
3340
3341	free_cpumask_var(iter->started);
3342	mutex_destroy(&iter->mutex);
3343	kfree(iter->trace);
3344	kfree(iter);
3345
 
 
3346	return 0;
3347}
3348
3349static unsigned int
3350tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3351{
3352	struct trace_iterator *iter = filp->private_data;
3353
3354	if (trace_flags & TRACE_ITER_BLOCK) {
 
 
 
 
3355		/*
3356		 * Always select as readable when in blocking mode
3357		 */
3358		return POLLIN | POLLRDNORM;
3359	} else {
3360		if (!trace_empty(iter))
3361			return POLLIN | POLLRDNORM;
3362		poll_wait(filp, &trace_wait, poll_table);
3363		if (!trace_empty(iter))
3364			return POLLIN | POLLRDNORM;
3365
3366		return 0;
3367	}
3368}
3369
3370
3371void default_wait_pipe(struct trace_iterator *iter)
3372{
3373	DEFINE_WAIT(wait);
3374
3375	prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3376
3377	if (trace_empty(iter))
3378		schedule();
3379
3380	finish_wait(&trace_wait, &wait);
3381}
3382
3383/*
3384 * This is a make-shift waitqueue.
3385 * A tracer might use this callback on some rare cases:
3386 *
3387 *  1) the current tracer might hold the runqueue lock when it wakes up
3388 *     a reader, hence a deadlock (sched, function, and function graph tracers)
3389 *  2) the function tracers, trace all functions, we don't want
3390 *     the overhead of calling wake_up and friends
3391 *     (and tracing them too)
3392 *
3393 *     Anyway, this is really very primitive wakeup.
3394 */
3395void poll_wait_pipe(struct trace_iterator *iter)
3396{
3397	set_current_state(TASK_INTERRUPTIBLE);
3398	/* sleep for 100 msecs, and try again. */
3399	schedule_timeout(HZ / 10);
3400}
3401
3402/* Must be called with trace_types_lock mutex held. */
3403static int tracing_wait_pipe(struct file *filp)
3404{
3405	struct trace_iterator *iter = filp->private_data;
 
3406
3407	while (trace_empty(iter)) {
3408
3409		if ((filp->f_flags & O_NONBLOCK)) {
3410			return -EAGAIN;
3411		}
3412
3413		mutex_unlock(&iter->mutex);
3414
3415		iter->trace->wait_pipe(iter);
3416
3417		mutex_lock(&iter->mutex);
3418
3419		if (signal_pending(current))
3420			return -EINTR;
3421
3422		/*
3423		 * We block until we read something and tracing is disabled.
3424		 * We still block if tracing is disabled, but we have never
3425		 * read anything. This allows a user to cat this file, and
3426		 * then enable tracing. But after we have read something,
3427		 * we give an EOF when tracing is again disabled.
3428		 *
3429		 * iter->pos will be 0 if we haven't read anything.
3430		 */
3431		if (!tracer_enabled && iter->pos)
3432			break;
 
 
 
 
 
 
 
 
 
3433	}
3434
3435	return 1;
3436}
3437
3438/*
3439 * Consumer reader.
3440 */
3441static ssize_t
3442tracing_read_pipe(struct file *filp, char __user *ubuf,
3443		  size_t cnt, loff_t *ppos)
3444{
3445	struct trace_iterator *iter = filp->private_data;
3446	static struct tracer *old_tracer;
3447	ssize_t sret;
3448
3449	/* return any leftover data */
3450	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3451	if (sret != -EBUSY)
3452		return sret;
3453
3454	trace_seq_init(&iter->seq);
3455
3456	/* copy the tracer to avoid using a global lock all around */
3457	mutex_lock(&trace_types_lock);
3458	if (unlikely(old_tracer != current_trace && current_trace)) {
3459		old_tracer = current_trace;
3460		*iter->trace = *current_trace;
3461	}
3462	mutex_unlock(&trace_types_lock);
3463
3464	/*
3465	 * Avoid more than one consumer on a single file descriptor
3466	 * This is just a matter of traces coherency, the ring buffer itself
3467	 * is protected.
3468	 */
3469	mutex_lock(&iter->mutex);
 
 
 
 
 
 
 
 
3470	if (iter->trace->read) {
3471		sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3472		if (sret)
3473			goto out;
3474	}
3475
3476waitagain:
3477	sret = tracing_wait_pipe(filp);
3478	if (sret <= 0)
3479		goto out;
3480
3481	/* stop when tracing is finished */
3482	if (trace_empty(iter)) {
3483		sret = 0;
3484		goto out;
3485	}
3486
3487	if (cnt >= PAGE_SIZE)
3488		cnt = PAGE_SIZE - 1;
3489
3490	/* reset all but tr, trace, and overruns */
3491	memset(&iter->seq, 0,
3492	       sizeof(struct trace_iterator) -
3493	       offsetof(struct trace_iterator, seq));
3494	iter->pos = -1;
3495
3496	trace_event_read_lock();
3497	trace_access_lock(iter->cpu_file);
3498	while (trace_find_next_entry_inc(iter) != NULL) {
3499		enum print_line_t ret;
3500		int len = iter->seq.len;
3501
3502		ret = print_trace_line(iter);
3503		if (ret == TRACE_TYPE_PARTIAL_LINE) {
3504			/* don't print partial lines */
3505			iter->seq.len = len;
 
 
 
 
 
 
 
 
 
 
 
 
 
3506			break;
3507		}
3508		if (ret != TRACE_TYPE_NO_CONSUME)
3509			trace_consume(iter);
3510
3511		if (iter->seq.len >= cnt)
3512			break;
3513
3514		/*
3515		 * Setting the full flag means we reached the trace_seq buffer
3516		 * size and we should leave by partial output condition above.
3517		 * One of the trace_seq_* functions is not used properly.
3518		 */
3519		WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3520			  iter->ent->type);
3521	}
3522	trace_access_unlock(iter->cpu_file);
3523	trace_event_read_unlock();
3524
3525	/* Now copy what we have to the user */
3526	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3527	if (iter->seq.readpos >= iter->seq.len)
3528		trace_seq_init(&iter->seq);
3529
3530	/*
3531	 * If there was nothing to send to user, in spite of consuming trace
3532	 * entries, go back to wait for more entries.
3533	 */
3534	if (sret == -EBUSY)
3535		goto waitagain;
3536
3537out:
3538	mutex_unlock(&iter->mutex);
3539
3540	return sret;
3541}
3542
3543static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3544				     struct pipe_buffer *buf)
3545{
3546	__free_page(buf->page);
3547}
3548
3549static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3550				     unsigned int idx)
3551{
3552	__free_page(spd->pages[idx]);
3553}
3554
3555static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3556	.can_merge		= 0,
3557	.map			= generic_pipe_buf_map,
3558	.unmap			= generic_pipe_buf_unmap,
3559	.confirm		= generic_pipe_buf_confirm,
3560	.release		= tracing_pipe_buf_release,
3561	.steal			= generic_pipe_buf_steal,
3562	.get			= generic_pipe_buf_get,
3563};
3564
3565static size_t
3566tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3567{
3568	size_t count;
 
3569	int ret;
3570
3571	/* Seq buffer is page-sized, exactly what we need. */
3572	for (;;) {
3573		count = iter->seq.len;
3574		ret = print_trace_line(iter);
3575		count = iter->seq.len - count;
3576		if (rem < count) {
3577			rem = 0;
3578			iter->seq.len -= count;
3579			break;
3580		}
 
 
 
 
 
 
3581		if (ret == TRACE_TYPE_PARTIAL_LINE) {
3582			iter->seq.len -= count;
 
 
 
 
 
 
 
3583			break;
3584		}
3585
3586		if (ret != TRACE_TYPE_NO_CONSUME)
3587			trace_consume(iter);
3588		rem -= count;
3589		if (!trace_find_next_entry_inc(iter))	{
3590			rem = 0;
3591			iter->ent = NULL;
3592			break;
3593		}
3594	}
3595
3596	return rem;
3597}
3598
3599static ssize_t tracing_splice_read_pipe(struct file *filp,
3600					loff_t *ppos,
3601					struct pipe_inode_info *pipe,
3602					size_t len,
3603					unsigned int flags)
3604{
3605	struct page *pages_def[PIPE_DEF_BUFFERS];
3606	struct partial_page partial_def[PIPE_DEF_BUFFERS];
3607	struct trace_iterator *iter = filp->private_data;
3608	struct splice_pipe_desc spd = {
3609		.pages		= pages_def,
3610		.partial	= partial_def,
3611		.nr_pages	= 0, /* This gets updated below. */
3612		.nr_pages_max	= PIPE_DEF_BUFFERS,
3613		.flags		= flags,
3614		.ops		= &tracing_pipe_buf_ops,
3615		.spd_release	= tracing_spd_release_pipe,
3616	};
3617	static struct tracer *old_tracer;
3618	ssize_t ret;
3619	size_t rem;
3620	unsigned int i;
3621
3622	if (splice_grow_spd(pipe, &spd))
3623		return -ENOMEM;
3624
3625	/* copy the tracer to avoid using a global lock all around */
3626	mutex_lock(&trace_types_lock);
3627	if (unlikely(old_tracer != current_trace && current_trace)) {
3628		old_tracer = current_trace;
3629		*iter->trace = *current_trace;
3630	}
3631	mutex_unlock(&trace_types_lock);
3632
3633	mutex_lock(&iter->mutex);
3634
3635	if (iter->trace->splice_read) {
3636		ret = iter->trace->splice_read(iter, filp,
3637					       ppos, pipe, len, flags);
3638		if (ret)
3639			goto out_err;
3640	}
3641
3642	ret = tracing_wait_pipe(filp);
3643	if (ret <= 0)
3644		goto out_err;
3645
3646	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3647		ret = -EFAULT;
3648		goto out_err;
3649	}
3650
3651	trace_event_read_lock();
3652	trace_access_lock(iter->cpu_file);
3653
3654	/* Fill as many pages as possible. */
3655	for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3656		spd.pages[i] = alloc_page(GFP_KERNEL);
3657		if (!spd.pages[i])
3658			break;
3659
3660		rem = tracing_fill_pipe_page(rem, iter);
3661
3662		/* Copy the data into the page, so we can start over. */
3663		ret = trace_seq_to_buffer(&iter->seq,
3664					  page_address(spd.pages[i]),
3665					  iter->seq.len);
3666		if (ret < 0) {
3667			__free_page(spd.pages[i]);
3668			break;
3669		}
3670		spd.partial[i].offset = 0;
3671		spd.partial[i].len = iter->seq.len;
3672
3673		trace_seq_init(&iter->seq);
3674	}
3675
3676	trace_access_unlock(iter->cpu_file);
3677	trace_event_read_unlock();
3678	mutex_unlock(&iter->mutex);
3679
3680	spd.nr_pages = i;
3681
3682	ret = splice_to_pipe(pipe, &spd);
 
 
 
3683out:
3684	splice_shrink_spd(&spd);
3685	return ret;
3686
3687out_err:
3688	mutex_unlock(&iter->mutex);
3689	goto out;
3690}
3691
3692struct ftrace_entries_info {
3693	struct trace_array	*tr;
3694	int			cpu;
3695};
3696
3697static int tracing_entries_open(struct inode *inode, struct file *filp)
3698{
3699	struct ftrace_entries_info *info;
3700
3701	if (tracing_disabled)
3702		return -ENODEV;
3703
3704	info = kzalloc(sizeof(*info), GFP_KERNEL);
3705	if (!info)
3706		return -ENOMEM;
3707
3708	info->tr = &global_trace;
3709	info->cpu = (unsigned long)inode->i_private;
3710
3711	filp->private_data = info;
3712
3713	return 0;
3714}
3715
3716static ssize_t
3717tracing_entries_read(struct file *filp, char __user *ubuf,
3718		     size_t cnt, loff_t *ppos)
3719{
3720	struct ftrace_entries_info *info = filp->private_data;
3721	struct trace_array *tr = info->tr;
 
3722	char buf[64];
3723	int r = 0;
3724	ssize_t ret;
3725
3726	mutex_lock(&trace_types_lock);
3727
3728	if (info->cpu == RING_BUFFER_ALL_CPUS) {
3729		int cpu, buf_size_same;
3730		unsigned long size;
3731
3732		size = 0;
3733		buf_size_same = 1;
3734		/* check if all cpu sizes are same */
3735		for_each_tracing_cpu(cpu) {
3736			/* fill in the size from first enabled cpu */
3737			if (size == 0)
3738				size = tr->data[cpu]->entries;
3739			if (size != tr->data[cpu]->entries) {
3740				buf_size_same = 0;
3741				break;
3742			}
3743		}
3744
3745		if (buf_size_same) {
3746			if (!ring_buffer_expanded)
3747				r = sprintf(buf, "%lu (expanded: %lu)\n",
3748					    size >> 10,
3749					    trace_buf_size >> 10);
3750			else
3751				r = sprintf(buf, "%lu\n", size >> 10);
3752		} else
3753			r = sprintf(buf, "X\n");
3754	} else
3755		r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3756
3757	mutex_unlock(&trace_types_lock);
3758
3759	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3760	return ret;
3761}
3762
3763static ssize_t
3764tracing_entries_write(struct file *filp, const char __user *ubuf,
3765		      size_t cnt, loff_t *ppos)
3766{
3767	struct ftrace_entries_info *info = filp->private_data;
 
3768	unsigned long val;
3769	int ret;
3770
3771	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3772	if (ret)
3773		return ret;
3774
3775	/* must have at least 1 entry */
3776	if (!val)
3777		return -EINVAL;
3778
3779	/* value is in KB */
3780	val <<= 10;
3781
3782	ret = tracing_resize_ring_buffer(val, info->cpu);
3783	if (ret < 0)
3784		return ret;
3785
3786	*ppos += cnt;
3787
3788	return cnt;
3789}
3790
3791static int
3792tracing_entries_release(struct inode *inode, struct file *filp)
3793{
3794	struct ftrace_entries_info *info = filp->private_data;
3795
3796	kfree(info);
3797
3798	return 0;
3799}
3800
3801static ssize_t
3802tracing_total_entries_read(struct file *filp, char __user *ubuf,
3803				size_t cnt, loff_t *ppos)
3804{
3805	struct trace_array *tr = filp->private_data;
3806	char buf[64];
3807	int r, cpu;
3808	unsigned long size = 0, expanded_size = 0;
3809
3810	mutex_lock(&trace_types_lock);
3811	for_each_tracing_cpu(cpu) {
3812		size += tr->data[cpu]->entries >> 10;
3813		if (!ring_buffer_expanded)
3814			expanded_size += trace_buf_size >> 10;
3815	}
3816	if (ring_buffer_expanded)
3817		r = sprintf(buf, "%lu\n", size);
3818	else
3819		r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3820	mutex_unlock(&trace_types_lock);
3821
3822	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3823}
3824
3825static ssize_t
3826tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3827			  size_t cnt, loff_t *ppos)
3828{
3829	/*
3830	 * There is no need to read what the user has written, this function
3831	 * is just to make sure that there is no error when "echo" is used
3832	 */
3833
3834	*ppos += cnt;
3835
3836	return cnt;
3837}
3838
3839static int
3840tracing_free_buffer_release(struct inode *inode, struct file *filp)
3841{
 
 
3842	/* disable tracing ? */
3843	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3844		tracing_off();
3845	/* resize the ring buffer to 0 */
3846	tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
 
 
3847
3848	return 0;
3849}
3850
 
 
3851static ssize_t
3852tracing_mark_write(struct file *filp, const char __user *ubuf,
3853					size_t cnt, loff_t *fpos)
3854{
3855	unsigned long addr = (unsigned long)ubuf;
3856	struct ring_buffer_event *event;
3857	struct ring_buffer *buffer;
 
3858	struct print_entry *entry;
3859	unsigned long irq_flags;
3860	struct page *pages[2];
3861	void *map_page[2];
3862	int nr_pages = 1;
3863	ssize_t written;
3864	int offset;
3865	int size;
3866	int len;
3867	int ret;
3868	int i;
 
 
3869
3870	if (tracing_disabled)
3871		return -EINVAL;
3872
3873	if (cnt > TRACE_BUF_SIZE)
3874		cnt = TRACE_BUF_SIZE;
3875
3876	/*
3877	 * Userspace is injecting traces into the kernel trace buffer.
3878	 * We want to be as non intrusive as possible.
3879	 * To do so, we do not want to allocate any special buffers
3880	 * or take any locks, but instead write the userspace data
3881	 * straight into the ring buffer.
3882	 *
3883	 * First we need to pin the userspace buffer into memory,
3884	 * which, most likely it is, because it just referenced it.
3885	 * But there's no guarantee that it is. By using get_user_pages_fast()
3886	 * and kmap_atomic/kunmap_atomic() we can get access to the
3887	 * pages directly. We then write the data directly into the
3888	 * ring buffer.
3889	 */
3890	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
3891
3892	/* check if we cross pages */
3893	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3894		nr_pages = 2;
3895
3896	offset = addr & (PAGE_SIZE - 1);
3897	addr &= PAGE_MASK;
3898
3899	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3900	if (ret < nr_pages) {
3901		while (--ret >= 0)
3902			put_page(pages[ret]);
3903		written = -EFAULT;
3904		goto out;
3905	}
3906
3907	for (i = 0; i < nr_pages; i++)
3908		map_page[i] = kmap_atomic(pages[i]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3909
3910	local_save_flags(irq_flags);
3911	size = sizeof(*entry) + cnt + 2; /* possible \n added */
3912	buffer = global_trace.buffer;
3913	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3914					  irq_flags, preempt_count());
3915	if (!event) {
3916		/* Ring buffer disabled, return as if not open for write */
3917		written = -EBADF;
3918		goto out_unlock;
3919	}
3920
3921	entry = ring_buffer_event_data(event);
3922	entry->ip = _THIS_IP_;
3923
3924	if (nr_pages == 2) {
3925		len = PAGE_SIZE - offset;
3926		memcpy(&entry->buf, map_page[0] + offset, len);
3927		memcpy(&entry->buf[len], map_page[1], cnt - len);
 
3928	} else
3929		memcpy(&entry->buf, map_page[0] + offset, cnt);
 
 
 
 
 
 
3930
3931	if (entry->buf[cnt - 1] != '\n') {
3932		entry->buf[cnt] = '\n';
3933		entry->buf[cnt + 1] = '\0';
3934	} else
3935		entry->buf[cnt] = '\0';
3936
3937	ring_buffer_unlock_commit(buffer, event);
 
 
3938
3939	written = cnt;
 
3940
3941	*fpos += written;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3942
3943 out_unlock:
3944	for (i = 0; i < nr_pages; i++){
3945		kunmap_atomic(map_page[i]);
3946		put_page(pages[i]);
3947	}
3948 out:
3949	return written;
3950}
3951
3952static int tracing_clock_show(struct seq_file *m, void *v)
3953{
 
3954	int i;
3955
3956	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3957		seq_printf(m,
3958			"%s%s%s%s", i ? " " : "",
3959			i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3960			i == trace_clock_id ? "]" : "");
3961	seq_putc(m, '\n');
3962
3963	return 0;
3964}
3965
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3966static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3967				   size_t cnt, loff_t *fpos)
3968{
 
 
3969	char buf[64];
3970	const char *clockstr;
3971	int i;
3972
3973	if (cnt >= sizeof(buf))
3974		return -EINVAL;
3975
3976	if (copy_from_user(&buf, ubuf, cnt))
3977		return -EFAULT;
3978
3979	buf[cnt] = 0;
3980
3981	clockstr = strstrip(buf);
3982
3983	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3984		if (strcmp(trace_clocks[i].name, clockstr) == 0)
3985			break;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3986	}
3987	if (i == ARRAY_SIZE(trace_clocks))
3988		return -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3989
3990	trace_clock_id = i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3991
3992	mutex_lock(&trace_types_lock);
3993
3994	ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3995	if (max_tr.buffer)
3996		ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3997
 
 
 
 
 
3998	mutex_unlock(&trace_types_lock);
 
 
3999
4000	*fpos += cnt;
 
 
 
4001
4002	return cnt;
 
 
 
 
 
 
 
 
 
 
4003}
4004
4005static int tracing_clock_open(struct inode *inode, struct file *file)
 
 
 
 
 
 
 
4006{
4007	if (tracing_disabled)
4008		return -ENODEV;
4009	return single_open(file, tracing_clock_show, NULL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4010}
4011
4012static const struct file_operations tracing_max_lat_fops = {
 
 
 
4013	.open		= tracing_open_generic,
4014	.read		= tracing_max_lat_read,
4015	.write		= tracing_max_lat_write,
4016	.llseek		= generic_file_llseek,
4017};
4018
4019static const struct file_operations tracing_ctrl_fops = {
4020	.open		= tracing_open_generic,
4021	.read		= tracing_ctrl_read,
4022	.write		= tracing_ctrl_write,
 
4023	.llseek		= generic_file_llseek,
 
4024};
 
4025
4026static const struct file_operations set_tracer_fops = {
4027	.open		= tracing_open_generic,
4028	.read		= tracing_set_trace_read,
4029	.write		= tracing_set_trace_write,
4030	.llseek		= generic_file_llseek,
 
4031};
4032
4033static const struct file_operations tracing_pipe_fops = {
4034	.open		= tracing_open_pipe,
4035	.poll		= tracing_poll_pipe,
4036	.read		= tracing_read_pipe,
4037	.splice_read	= tracing_splice_read_pipe,
4038	.release	= tracing_release_pipe,
4039	.llseek		= no_llseek,
4040};
4041
4042static const struct file_operations tracing_entries_fops = {
4043	.open		= tracing_entries_open,
4044	.read		= tracing_entries_read,
4045	.write		= tracing_entries_write,
4046	.release	= tracing_entries_release,
4047	.llseek		= generic_file_llseek,
 
4048};
4049
4050static const struct file_operations tracing_total_entries_fops = {
4051	.open		= tracing_open_generic,
4052	.read		= tracing_total_entries_read,
4053	.llseek		= generic_file_llseek,
 
4054};
4055
4056static const struct file_operations tracing_free_buffer_fops = {
 
4057	.write		= tracing_free_buffer_write,
4058	.release	= tracing_free_buffer_release,
4059};
4060
4061static const struct file_operations tracing_mark_fops = {
4062	.open		= tracing_open_generic,
4063	.write		= tracing_mark_write,
4064	.llseek		= generic_file_llseek,
 
 
 
 
 
 
4065};
4066
4067static const struct file_operations trace_clock_fops = {
4068	.open		= tracing_clock_open,
4069	.read		= seq_read,
4070	.llseek		= seq_lseek,
4071	.release	= single_release,
4072	.write		= tracing_clock_write,
4073};
4074
4075struct ftrace_buffer_info {
4076	struct trace_array	*tr;
4077	void			*spare;
4078	int			cpu;
4079	unsigned int		read;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4080};
4081
4082static int tracing_buffers_open(struct inode *inode, struct file *filp)
4083{
4084	int cpu = (int)(long)inode->i_private;
4085	struct ftrace_buffer_info *info;
 
4086
4087	if (tracing_disabled)
4088		return -ENODEV;
 
4089
4090	info = kzalloc(sizeof(*info), GFP_KERNEL);
4091	if (!info)
 
4092		return -ENOMEM;
 
4093
4094	info->tr	= &global_trace;
4095	info->cpu	= cpu;
4096	info->spare	= NULL;
 
 
 
 
4097	/* Force reading ring buffer for first read */
4098	info->read	= (unsigned int)-1;
4099
4100	filp->private_data = info;
4101
4102	return nonseekable_open(inode, filp);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4103}
4104
4105static ssize_t
4106tracing_buffers_read(struct file *filp, char __user *ubuf,
4107		     size_t count, loff_t *ppos)
4108{
4109	struct ftrace_buffer_info *info = filp->private_data;
4110	ssize_t ret;
4111	size_t size;
 
 
 
4112
4113	if (!count)
4114		return 0;
4115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4116	if (!info->spare)
4117		info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4118	if (!info->spare)
4119		return -ENOMEM;
4120
4121	/* Do we have previous read data to read? */
4122	if (info->read < PAGE_SIZE)
4123		goto read;
4124
4125	trace_access_lock(info->cpu);
4126	ret = ring_buffer_read_page(info->tr->buffer,
4127				    &info->spare,
 
4128				    count,
4129				    info->cpu, 0);
4130	trace_access_unlock(info->cpu);
4131	if (ret < 0)
 
 
 
 
 
 
 
 
 
 
 
4132		return 0;
 
4133
4134	info->read = 0;
4135
4136read:
4137	size = PAGE_SIZE - info->read;
4138	if (size > count)
4139		size = count;
4140
4141	ret = copy_to_user(ubuf, info->spare + info->read, size);
4142	if (ret == size)
4143		return -EFAULT;
 
4144	size -= ret;
4145
4146	*ppos += size;
4147	info->read += size;
4148
4149	return size;
4150}
4151
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4152static int tracing_buffers_release(struct inode *inode, struct file *file)
4153{
4154	struct ftrace_buffer_info *info = file->private_data;
 
 
 
 
 
 
 
4155
4156	if (info->spare)
4157		ring_buffer_free_read_page(info->tr->buffer, info->spare);
4158	kfree(info);
 
 
 
4159
4160	return 0;
4161}
4162
4163struct buffer_ref {
4164	struct ring_buffer	*buffer;
4165	void			*page;
4166	int			ref;
 
4167};
4168
4169static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4170				    struct pipe_buffer *buf)
4171{
4172	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4173
4174	if (--ref->ref)
4175		return;
4176
4177	ring_buffer_free_read_page(ref->buffer, ref->page);
4178	kfree(ref);
4179	buf->private = 0;
4180}
4181
4182static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
4183				 struct pipe_buffer *buf)
4184{
4185	return 1;
 
 
 
4186}
4187
4188static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4189				struct pipe_buffer *buf)
4190{
4191	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4192
4193	ref->ref++;
 
 
 
 
4194}
4195
4196/* Pipe buffer operations for a buffer. */
4197static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4198	.can_merge		= 0,
4199	.map			= generic_pipe_buf_map,
4200	.unmap			= generic_pipe_buf_unmap,
4201	.confirm		= generic_pipe_buf_confirm,
4202	.release		= buffer_pipe_buf_release,
4203	.steal			= buffer_pipe_buf_steal,
4204	.get			= buffer_pipe_buf_get,
4205};
4206
4207/*
4208 * Callback from splice_to_pipe(), if we need to release some pages
4209 * at the end of the spd in case we error'ed out in filling the pipe.
4210 */
4211static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4212{
4213	struct buffer_ref *ref =
4214		(struct buffer_ref *)spd->partial[i].private;
4215
4216	if (--ref->ref)
4217		return;
4218
4219	ring_buffer_free_read_page(ref->buffer, ref->page);
4220	kfree(ref);
4221	spd->partial[i].private = 0;
4222}
4223
4224static ssize_t
4225tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4226			    struct pipe_inode_info *pipe, size_t len,
4227			    unsigned int flags)
4228{
4229	struct ftrace_buffer_info *info = file->private_data;
 
4230	struct partial_page partial_def[PIPE_DEF_BUFFERS];
4231	struct page *pages_def[PIPE_DEF_BUFFERS];
4232	struct splice_pipe_desc spd = {
4233		.pages		= pages_def,
4234		.partial	= partial_def,
4235		.nr_pages_max	= PIPE_DEF_BUFFERS,
4236		.flags		= flags,
4237		.ops		= &buffer_pipe_buf_ops,
4238		.spd_release	= buffer_spd_release,
4239	};
4240	struct buffer_ref *ref;
4241	int entries, size, i;
4242	size_t ret;
 
 
4243
4244	if (splice_grow_spd(pipe, &spd))
4245		return -ENOMEM;
 
 
4246
4247	if (*ppos & (PAGE_SIZE - 1)) {
4248		WARN_ONCE(1, "Ftrace: previous read must page-align\n");
4249		ret = -EINVAL;
4250		goto out;
4251	}
4252
4253	if (len & (PAGE_SIZE - 1)) {
4254		WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
4255		if (len < PAGE_SIZE) {
4256			ret = -EINVAL;
4257			goto out;
4258		}
4259		len &= PAGE_MASK;
4260	}
4261
4262	trace_access_lock(info->cpu);
4263	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
 
 
 
 
4264
4265	for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4266		struct page *page;
4267		int r;
4268
4269		ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4270		if (!ref)
 
4271			break;
 
4272
4273		ref->ref = 1;
4274		ref->buffer = info->tr->buffer;
4275		ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4276		if (!ref->page) {
 
 
4277			kfree(ref);
4278			break;
4279		}
 
4280
4281		r = ring_buffer_read_page(ref->buffer, &ref->page,
4282					  len, info->cpu, 1);
4283		if (r < 0) {
4284			ring_buffer_free_read_page(ref->buffer, ref->page);
 
4285			kfree(ref);
4286			break;
4287		}
4288
4289		/*
4290		 * zero out any left over data, this is going to
4291		 * user land.
4292		 */
4293		size = ring_buffer_page_len(ref->page);
4294		if (size < PAGE_SIZE)
4295			memset(ref->page + size, 0, PAGE_SIZE - size);
4296
4297		page = virt_to_page(ref->page);
4298
4299		spd.pages[i] = page;
4300		spd.partial[i].len = PAGE_SIZE;
4301		spd.partial[i].offset = 0;
4302		spd.partial[i].private = (unsigned long)ref;
4303		spd.nr_pages++;
4304		*ppos += PAGE_SIZE;
4305
4306		entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4307	}
4308
4309	trace_access_unlock(info->cpu);
4310	spd.nr_pages = i;
4311
4312	/* did we read anything? */
4313	if (!spd.nr_pages) {
4314		if (flags & SPLICE_F_NONBLOCK)
4315			ret = -EAGAIN;
4316		else
4317			ret = 0;
4318		/* TODO: block */
4319		goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4320	}
4321
4322	ret = splice_to_pipe(pipe, &spd);
4323	splice_shrink_spd(&spd);
4324out:
 
 
4325	return ret;
4326}
4327
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4328static const struct file_operations tracing_buffers_fops = {
4329	.open		= tracing_buffers_open,
4330	.read		= tracing_buffers_read,
 
4331	.release	= tracing_buffers_release,
 
4332	.splice_read	= tracing_buffers_splice_read,
 
4333	.llseek		= no_llseek,
4334};
4335
4336static ssize_t
4337tracing_stats_read(struct file *filp, char __user *ubuf,
4338		   size_t count, loff_t *ppos)
4339{
4340	unsigned long cpu = (unsigned long)filp->private_data;
4341	struct trace_array *tr = &global_trace;
 
 
4342	struct trace_seq *s;
4343	unsigned long cnt;
4344	unsigned long long t;
4345	unsigned long usec_rem;
4346
4347	s = kmalloc(sizeof(*s), GFP_KERNEL);
4348	if (!s)
4349		return -ENOMEM;
4350
4351	trace_seq_init(s);
4352
4353	cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4354	trace_seq_printf(s, "entries: %ld\n", cnt);
4355
4356	cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4357	trace_seq_printf(s, "overrun: %ld\n", cnt);
4358
4359	cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4360	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4361
4362	cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4363	trace_seq_printf(s, "bytes: %ld\n", cnt);
4364
4365	t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4366	usec_rem = do_div(t, USEC_PER_SEC);
4367	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
4368
4369	t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4370	usec_rem = do_div(t, USEC_PER_SEC);
4371	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4372
4373	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
 
 
 
 
4374
4375	kfree(s);
4376
4377	return count;
4378}
4379
4380static const struct file_operations tracing_stats_fops = {
4381	.open		= tracing_open_generic,
4382	.read		= tracing_stats_read,
4383	.llseek		= generic_file_llseek,
 
4384};
4385
4386#ifdef CONFIG_DYNAMIC_FTRACE
4387
4388int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4389{
4390	return 0;
4391}
4392
4393static ssize_t
4394tracing_read_dyn_info(struct file *filp, char __user *ubuf,
4395		  size_t cnt, loff_t *ppos)
4396{
4397	static char ftrace_dyn_info_buffer[1024];
4398	static DEFINE_MUTEX(dyn_info_mutex);
4399	unsigned long *p = filp->private_data;
4400	char *buf = ftrace_dyn_info_buffer;
4401	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4402	int r;
4403
4404	mutex_lock(&dyn_info_mutex);
4405	r = sprintf(buf, "%ld ", *p);
4406
4407	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4408	buf[r++] = '\n';
4409
4410	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4411
4412	mutex_unlock(&dyn_info_mutex);
 
 
 
4413
4414	return r;
 
 
4415}
4416
4417static const struct file_operations tracing_dyn_info_fops = {
4418	.open		= tracing_open_generic,
4419	.read		= tracing_read_dyn_info,
4420	.llseek		= generic_file_llseek,
4421};
4422#endif
4423
4424static struct dentry *d_tracer;
 
 
 
 
 
 
 
4425
4426struct dentry *tracing_init_dentry(void)
 
 
 
4427{
4428	static int once;
 
4429
4430	if (d_tracer)
4431		return d_tracer;
4432
4433	if (!debugfs_initialized())
4434		return NULL;
4435
4436	d_tracer = debugfs_create_dir("tracing", NULL);
 
4437
4438	if (!d_tracer && !once) {
4439		once = 1;
4440		pr_warning("Could not create debugfs directory 'tracing'\n");
4441		return NULL;
4442	}
4443
4444	return d_tracer;
4445}
4446
4447static struct dentry *d_percpu;
 
 
 
 
 
 
 
 
 
 
 
 
4448
4449struct dentry *tracing_dentry_percpu(void)
 
 
 
 
 
 
 
 
 
 
4450{
4451	static int once;
4452	struct dentry *d_tracer;
4453
4454	if (d_percpu)
4455		return d_percpu;
 
 
 
 
 
 
 
4456
4457	d_tracer = tracing_init_dentry();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4458
4459	if (!d_tracer)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4460		return NULL;
4461
4462	d_percpu = debugfs_create_dir("per_cpu", d_tracer);
 
 
4463
4464	if (!d_percpu && !once) {
4465		once = 1;
4466		pr_warning("Could not create debugfs directory 'per_cpu'\n");
 
 
 
 
 
 
4467		return NULL;
4468	}
4469
4470	return d_percpu;
 
 
 
 
 
4471}
4472
4473static void tracing_init_debugfs_percpu(long cpu)
 
 
4474{
4475	struct dentry *d_percpu = tracing_dentry_percpu();
 
 
 
 
 
 
 
 
 
 
4476	struct dentry *d_cpu;
4477	char cpu_dir[30]; /* 30 characters should be more than enough */
4478
4479	if (!d_percpu)
4480		return;
4481
4482	snprintf(cpu_dir, 30, "cpu%ld", cpu);
4483	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4484	if (!d_cpu) {
4485		pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4486		return;
4487	}
4488
4489	/* per cpu trace_pipe */
4490	trace_create_file("trace_pipe", 0444, d_cpu,
4491			(void *) cpu, &tracing_pipe_fops);
4492
4493	/* per cpu trace */
4494	trace_create_file("trace", 0644, d_cpu,
4495			(void *) cpu, &tracing_fops);
 
 
 
4496
4497	trace_create_file("trace_pipe_raw", 0444, d_cpu,
4498			(void *) cpu, &tracing_buffers_fops);
4499
4500	trace_create_file("stats", 0444, d_cpu,
4501			(void *) cpu, &tracing_stats_fops);
4502
4503	trace_create_file("buffer_size_kb", 0444, d_cpu,
4504			(void *) cpu, &tracing_entries_fops);
 
 
 
 
 
4505}
4506
4507#ifdef CONFIG_FTRACE_SELFTEST
4508/* Let selftest have access to static functions in this file */
4509#include "trace_selftest.c"
4510#endif
4511
4512struct trace_option_dentry {
4513	struct tracer_opt		*opt;
4514	struct tracer_flags		*flags;
4515	struct dentry			*entry;
4516};
4517
4518static ssize_t
4519trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4520			loff_t *ppos)
4521{
4522	struct trace_option_dentry *topt = filp->private_data;
4523	char *buf;
4524
4525	if (topt->flags->val & topt->opt->bit)
4526		buf = "1\n";
4527	else
4528		buf = "0\n";
4529
4530	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4531}
4532
4533static ssize_t
4534trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4535			 loff_t *ppos)
4536{
4537	struct trace_option_dentry *topt = filp->private_data;
4538	unsigned long val;
4539	int ret;
4540
4541	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4542	if (ret)
4543		return ret;
4544
4545	if (val != 0 && val != 1)
4546		return -EINVAL;
4547
4548	if (!!(topt->flags->val & topt->opt->bit) != val) {
4549		mutex_lock(&trace_types_lock);
4550		ret = __set_tracer_option(current_trace, topt->flags,
4551					  topt->opt, !val);
4552		mutex_unlock(&trace_types_lock);
4553		if (ret)
4554			return ret;
4555	}
4556
4557	*ppos += cnt;
4558
4559	return cnt;
4560}
4561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4562
4563static const struct file_operations trace_options_fops = {
4564	.open = tracing_open_generic,
4565	.read = trace_options_read,
4566	.write = trace_options_write,
4567	.llseek	= generic_file_llseek,
 
4568};
4569
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4570static ssize_t
4571trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4572			loff_t *ppos)
4573{
4574	long index = (long)filp->private_data;
 
 
4575	char *buf;
4576
4577	if (trace_flags & (1 << index))
 
 
4578		buf = "1\n";
4579	else
4580		buf = "0\n";
4581
4582	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4583}
4584
4585static ssize_t
4586trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4587			 loff_t *ppos)
4588{
4589	long index = (long)filp->private_data;
 
 
4590	unsigned long val;
4591	int ret;
4592
 
 
4593	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4594	if (ret)
4595		return ret;
4596
4597	if (val != 0 && val != 1)
4598		return -EINVAL;
4599	set_tracer_flags(1 << index, val);
 
 
 
 
 
 
 
 
4600
4601	*ppos += cnt;
4602
4603	return cnt;
4604}
4605
4606static const struct file_operations trace_options_core_fops = {
4607	.open = tracing_open_generic,
4608	.read = trace_options_core_read,
4609	.write = trace_options_core_write,
4610	.llseek = generic_file_llseek,
4611};
4612
4613struct dentry *trace_create_file(const char *name,
4614				 umode_t mode,
4615				 struct dentry *parent,
4616				 void *data,
4617				 const struct file_operations *fops)
4618{
4619	struct dentry *ret;
4620
4621	ret = debugfs_create_file(name, mode, parent, data, fops);
4622	if (!ret)
4623		pr_warning("Could not create debugfs '%s' entry\n", name);
4624
4625	return ret;
4626}
4627
4628
4629static struct dentry *trace_options_init_dentry(void)
4630{
4631	struct dentry *d_tracer;
4632	static struct dentry *t_options;
4633
4634	if (t_options)
4635		return t_options;
4636
4637	d_tracer = tracing_init_dentry();
4638	if (!d_tracer)
4639		return NULL;
4640
4641	t_options = debugfs_create_dir("options", d_tracer);
4642	if (!t_options) {
4643		pr_warning("Could not create debugfs directory 'options'\n");
4644		return NULL;
4645	}
4646
4647	return t_options;
4648}
4649
4650static void
4651create_trace_option_file(struct trace_option_dentry *topt,
 
4652			 struct tracer_flags *flags,
4653			 struct tracer_opt *opt)
4654{
4655	struct dentry *t_options;
4656
4657	t_options = trace_options_init_dentry();
4658	if (!t_options)
4659		return;
4660
4661	topt->flags = flags;
4662	topt->opt = opt;
 
4663
4664	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4665				    &trace_options_fops);
4666
4667}
4668
4669static struct trace_option_dentry *
4670create_trace_option_files(struct tracer *tracer)
4671{
4672	struct trace_option_dentry *topts;
 
4673	struct tracer_flags *flags;
4674	struct tracer_opt *opts;
4675	int cnt;
 
4676
4677	if (!tracer)
4678		return NULL;
4679
4680	flags = tracer->flags;
4681
4682	if (!flags || !flags->opts)
4683		return NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
4684
4685	opts = flags->opts;
4686
4687	for (cnt = 0; opts[cnt].name; cnt++)
4688		;
4689
4690	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4691	if (!topts)
4692		return NULL;
4693
4694	for (cnt = 0; opts[cnt].name; cnt++)
4695		create_trace_option_file(&topts[cnt], flags,
4696					 &opts[cnt]);
4697
4698	return topts;
4699}
4700
4701static void
4702destroy_trace_option_files(struct trace_option_dentry *topts)
4703{
4704	int cnt;
4705
4706	if (!topts)
4707		return;
4708
4709	for (cnt = 0; topts[cnt].opt; cnt++) {
4710		if (topts[cnt].entry)
4711			debugfs_remove(topts[cnt].entry);
 
 
4712	}
4713
4714	kfree(topts);
 
 
 
 
 
 
 
 
 
 
 
4715}
4716
4717static struct dentry *
4718create_trace_option_core_file(const char *option, long index)
 
4719{
4720	struct dentry *t_options;
4721
4722	t_options = trace_options_init_dentry();
4723	if (!t_options)
4724		return NULL;
4725
4726	return trace_create_file(option, 0644, t_options, (void *)index,
4727				    &trace_options_core_fops);
 
4728}
4729
4730static __init void create_trace_options_dir(void)
4731{
4732	struct dentry *t_options;
 
4733	int i;
4734
4735	t_options = trace_options_init_dentry();
4736	if (!t_options)
4737		return;
4738
4739	for (i = 0; trace_options[i]; i++)
4740		create_trace_option_core_file(trace_options[i], i);
 
 
 
4741}
4742
4743static ssize_t
4744rb_simple_read(struct file *filp, char __user *ubuf,
4745	       size_t cnt, loff_t *ppos)
4746{
4747	struct trace_array *tr = filp->private_data;
4748	struct ring_buffer *buffer = tr->buffer;
4749	char buf[64];
4750	int r;
4751
4752	if (buffer)
4753		r = ring_buffer_record_is_on(buffer);
4754	else
4755		r = 0;
4756
4757	r = sprintf(buf, "%d\n", r);
4758
4759	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4760}
4761
4762static ssize_t
4763rb_simple_write(struct file *filp, const char __user *ubuf,
4764		size_t cnt, loff_t *ppos)
4765{
4766	struct trace_array *tr = filp->private_data;
4767	struct ring_buffer *buffer = tr->buffer;
4768	unsigned long val;
4769	int ret;
4770
4771	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4772	if (ret)
4773		return ret;
4774
4775	if (buffer) {
4776		if (val)
4777			ring_buffer_record_on(buffer);
4778		else
4779			ring_buffer_record_off(buffer);
 
 
 
 
 
 
 
 
 
 
 
4780	}
4781
4782	(*ppos)++;
4783
4784	return cnt;
4785}
4786
4787static const struct file_operations rb_simple_fops = {
4788	.open		= tracing_open_generic,
4789	.read		= rb_simple_read,
4790	.write		= rb_simple_write,
 
4791	.llseek		= default_llseek,
4792};
4793
4794static __init int tracer_init_debugfs(void)
 
 
4795{
4796	struct dentry *d_tracer;
4797	int cpu;
 
4798
4799	trace_access_lock_init();
 
4800
4801	d_tracer = tracing_init_dentry();
 
4802
4803	trace_create_file("tracing_enabled", 0644, d_tracer,
4804			&global_trace, &tracing_ctrl_fops);
 
 
 
 
 
4805
4806	trace_create_file("trace_options", 0644, d_tracer,
4807			NULL, &tracing_iter_fops);
 
4808
4809	trace_create_file("tracing_cpumask", 0644, d_tracer,
4810			NULL, &tracing_cpumask_fops);
4811
4812	trace_create_file("trace", 0644, d_tracer,
4813			(void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4814
4815	trace_create_file("available_tracers", 0444, d_tracer,
4816			&global_trace, &show_traces_fops);
4817
4818	trace_create_file("current_tracer", 0644, d_tracer,
4819			&global_trace, &set_tracer_fops);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4820
4821#ifdef CONFIG_TRACER_MAX_TRACE
4822	trace_create_file("tracing_max_latency", 0644, d_tracer,
4823			&tracing_max_latency, &tracing_max_lat_fops);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4824#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4825
4826	trace_create_file("tracing_thresh", 0644, d_tracer,
4827			&tracing_thresh, &tracing_max_lat_fops);
4828
4829	trace_create_file("README", 0444, d_tracer,
4830			NULL, &tracing_readme_fops);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4831
4832	trace_create_file("trace_pipe", 0444, d_tracer,
4833			(void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4834
4835	trace_create_file("buffer_size_kb", 0644, d_tracer,
4836			(void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4837
4838	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4839			&global_trace, &tracing_total_entries_fops);
4840
4841	trace_create_file("free_buffer", 0644, d_tracer,
4842			&global_trace, &tracing_free_buffer_fops);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4843
4844	trace_create_file("trace_marker", 0220, d_tracer,
4845			NULL, &tracing_mark_fops);
4846
4847	trace_create_file("saved_cmdlines", 0444, d_tracer,
4848			NULL, &tracing_saved_cmdlines_fops);
4849
4850	trace_create_file("trace_clock", 0644, d_tracer, NULL,
 
 
 
4851			  &trace_clock_fops);
4852
4853	trace_create_file("tracing_on", 0644, d_tracer,
4854			    &global_trace, &rb_simple_fops);
4855
4856#ifdef CONFIG_DYNAMIC_FTRACE
4857	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4858			&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4859#endif
4860
4861	create_trace_options_dir();
 
4862
4863	for_each_tracing_cpu(cpu)
4864		tracing_init_debugfs_percpu(cpu);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4865
4866	return 0;
4867}
4868
4869static int trace_panic_handler(struct notifier_block *this,
4870			       unsigned long event, void *unused)
 
 
 
 
 
 
4871{
4872	if (ftrace_dump_on_oops)
4873		ftrace_dump(ftrace_dump_on_oops);
4874	return NOTIFY_OK;
 
4875}
4876
4877static struct notifier_block trace_panic_notifier = {
4878	.notifier_call  = trace_panic_handler,
4879	.next           = NULL,
4880	.priority       = 150   /* priority: INT_MAX >= x >= 0 */
4881};
 
 
 
 
 
 
4882
4883static int trace_die_handler(struct notifier_block *self,
4884			     unsigned long val,
4885			     void *data)
 
 
 
 
4886{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4887	switch (val) {
4888	case DIE_OOPS:
4889		if (ftrace_dump_on_oops)
4890			ftrace_dump(ftrace_dump_on_oops);
4891		break;
4892	default:
 
4893		break;
4894	}
 
4895	return NOTIFY_OK;
4896}
4897
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4898static struct notifier_block trace_die_notifier = {
4899	.notifier_call = trace_die_handler,
4900	.priority = 200
4901};
4902
4903/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4904 * printk is set to max of 1024, we really don't need it that big.
4905 * Nothing should be printing 1000 characters anyway.
4906 */
4907#define TRACE_MAX_PRINT		1000
4908
4909/*
4910 * Define here KERN_TRACE so that we have one place to modify
4911 * it if we decide to change what log level the ftrace dump
4912 * should be at.
4913 */
4914#define KERN_TRACE		KERN_EMERG
4915
4916void
4917trace_printk_seq(struct trace_seq *s)
4918{
4919	/* Probably should print a warning here. */
4920	if (s->len >= 1000)
4921		s->len = 1000;
 
 
 
 
 
 
 
 
4922
4923	/* should be zero ended, but we are paranoid. */
4924	s->buffer[s->len] = 0;
4925
4926	printk(KERN_TRACE "%s", s->buffer);
4927
4928	trace_seq_init(s);
4929}
4930
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4931void trace_init_global_iter(struct trace_iterator *iter)
4932{
4933	iter->tr = &global_trace;
4934	iter->trace = current_trace;
4935	iter->cpu_file = TRACE_PIPE_ALL_CPU;
4936}
4937
4938static void
4939__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4940{
4941	static arch_spinlock_t ftrace_dump_lock =
4942		(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4943	/* use static because iter can be a bit big for the stack */
4944	static struct trace_iterator iter;
4945	unsigned int old_userobj;
4946	static int dump_ran;
4947	unsigned long flags;
4948	int cnt = 0, cpu;
4949
4950	/* only one dump */
4951	local_irq_save(flags);
4952	arch_spin_lock(&ftrace_dump_lock);
4953	if (dump_ran)
4954		goto out;
4955
4956	dump_ran = 1;
4957
4958	tracing_off();
4959
4960	/* Did function tracer already get disabled? */
4961	if (ftrace_is_dead()) {
4962		printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
4963		printk("#          MAY BE MISSING FUNCTION EVENTS\n");
4964	}
4965
4966	if (disable_tracing)
4967		ftrace_kill();
4968
4969	trace_init_global_iter(&iter);
 
4970
4971	for_each_tracing_cpu(cpu) {
4972		atomic_inc(&iter.tr->data[cpu]->disabled);
4973	}
4974
4975	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4976
4977	/* don't look at user memory in panic mode */
4978	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4979
4980	/* Simulate the iterator */
4981	iter.tr = &global_trace;
4982	iter.trace = current_trace;
4983
4984	switch (oops_dump_mode) {
4985	case DUMP_ALL:
4986		iter.cpu_file = TRACE_PIPE_ALL_CPU;
4987		break;
4988	case DUMP_ORIG:
4989		iter.cpu_file = raw_smp_processor_id();
4990		break;
4991	case DUMP_NONE:
4992		goto out_enable;
4993	default:
4994		printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4995		iter.cpu_file = TRACE_PIPE_ALL_CPU;
4996	}
4997
4998	printk(KERN_TRACE "Dumping ftrace buffer:\n");
 
 
 
 
 
 
 
 
 
4999
5000	/*
5001	 * We need to stop all tracing on all CPUS to read the
5002	 * the next buffer. This is a bit expensive, but is
5003	 * not done often. We fill all what we can read,
5004	 * and then release the locks again.
5005	 */
5006
5007	while (!trace_empty(&iter)) {
5008
5009		if (!cnt)
5010			printk(KERN_TRACE "---------------------------------\n");
5011
5012		cnt++;
5013
5014		/* reset all but tr, trace, and overruns */
5015		memset(&iter.seq, 0,
5016		       sizeof(struct trace_iterator) -
5017		       offsetof(struct trace_iterator, seq));
5018		iter.iter_flags |= TRACE_FILE_LAT_FMT;
5019		iter.pos = -1;
5020
5021		if (trace_find_next_entry_inc(&iter) != NULL) {
5022			int ret;
5023
5024			ret = print_trace_line(&iter);
5025			if (ret != TRACE_TYPE_NO_CONSUME)
5026				trace_consume(&iter);
5027		}
5028		touch_nmi_watchdog();
5029
5030		trace_printk_seq(&iter.seq);
5031	}
5032
5033	if (!cnt)
5034		printk(KERN_TRACE "   (ftrace buffer empty)\n");
5035	else
5036		printk(KERN_TRACE "---------------------------------\n");
5037
5038 out_enable:
5039	/* Re-enable tracing if requested */
5040	if (!disable_tracing) {
5041		trace_flags |= old_userobj;
5042
5043		for_each_tracing_cpu(cpu) {
5044			atomic_dec(&iter.tr->data[cpu]->disabled);
5045		}
5046		tracing_on();
5047	}
5048
5049 out:
5050	arch_spin_unlock(&ftrace_dump_lock);
5051	local_irq_restore(flags);
5052}
5053
5054/* By default: disable tracing after the dump */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5055void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5056{
5057	__ftrace_dump(true, oops_dump_mode);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5058}
5059EXPORT_SYMBOL_GPL(ftrace_dump);
5060
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5061__init static int tracer_alloc_buffers(void)
5062{
5063	int ring_buf_size;
5064	enum ring_buffer_flags rb_flags;
5065	int i;
5066	int ret = -ENOMEM;
5067
5068
 
 
 
 
 
 
 
 
 
 
 
5069	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5070		goto out;
5071
5072	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5073		goto out_free_buffer_mask;
5074
5075	/* Only allocate trace_printk buffers if a trace_printk exists */
5076	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
 
5077		trace_printk_init_buffers();
5078
5079	/* To save memory, keep the ring buffer size to its minimum */
5080	if (ring_buffer_expanded)
5081		ring_buf_size = trace_buf_size;
5082	else
5083		ring_buf_size = 1;
5084
5085	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5086
5087	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5088	cpumask_copy(tracing_cpumask, cpu_all_mask);
5089
5090	/* TODO: make the number of buffers hot pluggable with CPUS */
5091	global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5092	if (!global_trace.buffer) {
5093		printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5094		WARN_ON(1);
 
 
 
 
 
 
 
5095		goto out_free_cpumask;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5096	}
5097	if (global_trace.buffer_disabled)
5098		tracing_off();
5099
5100
5101#ifdef CONFIG_TRACER_MAX_TRACE
5102	max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5103	if (!max_tr.buffer) {
5104		printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5105		WARN_ON(1);
5106		ring_buffer_free(global_trace.buffer);
5107		goto out_free_cpumask;
5108	}
5109#endif
5110
5111	/* Allocate the first page for all buffers */
5112	for_each_tracing_cpu(i) {
5113		global_trace.data[i] = &per_cpu(global_trace_cpu, i);
5114		max_tr.data[i] = &per_cpu(max_tr_data, i);
5115	}
 
5116
5117	set_buffer_entries(&global_trace,
5118			   ring_buffer_size(global_trace.buffer, 0));
5119#ifdef CONFIG_TRACER_MAX_TRACE
5120	set_buffer_entries(&max_tr, 1);
5121#endif
 
5122
5123	trace_init_cmdlines();
5124
5125	register_tracer(&nop_trace);
5126	current_trace = &nop_trace;
 
 
 
5127	/* All seems OK, enable tracing */
5128	tracing_disabled = 0;
5129
5130	atomic_notifier_chain_register(&panic_notifier_list,
5131				       &trace_panic_notifier);
5132
5133	register_die_notifier(&trace_die_notifier);
5134
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5135	return 0;
5136
 
 
 
 
 
 
 
 
5137out_free_cpumask:
5138	free_cpumask_var(tracing_cpumask);
5139out_free_buffer_mask:
5140	free_cpumask_var(tracing_buffer_mask);
5141out:
5142	return ret;
5143}
5144
5145__init static int clear_boot_tracer(void)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5146{
5147	/*
5148	 * The default tracer at boot buffer is an init section.
5149	 * This function is called in lateinit. If we did not
5150	 * find the boot tracer, then clear it out, to prevent
5151	 * later registration from accessing the buffer that is
5152	 * about to be freed.
5153	 */
5154	if (!default_bootup_tracer)
5155		return 0;
5156
5157	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5158	       default_bootup_tracer);
5159	default_bootup_tracer = NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5160
 
 
5161	return 0;
5162}
5163
5164early_initcall(tracer_alloc_buffers);
5165fs_initcall(tracer_init_debugfs);
5166late_initcall(clear_boot_tracer);