1/*
   2 * ring buffer based function tracer
   3 *
   4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
   5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6 *
   7 * Originally taken from the RT patch by:
   8 *    Arnaldo Carvalho de Melo <acme@redhat.com>
   9 *
  10 * Based on code from the latency_tracer, that is:
  11 *  Copyright (C) 2004-2006 Ingo Molnar
  12 *  Copyright (C) 2004 Nadia Yvette Chambers
  13 */
  14#include <linux/ring_buffer.h>
  15#include <generated/utsrelease.h>
  16#include <linux/stacktrace.h>
  17#include <linux/writeback.h>
  18#include <linux/kallsyms.h>
  19#include <linux/seq_file.h>
  20#include <linux/notifier.h>
  21#include <linux/irqflags.h>
  22#include <linux/debugfs.h>
  23#include <linux/tracefs.h>
  24#include <linux/pagemap.h>
  25#include <linux/hardirq.h>
  26#include <linux/linkage.h>
  27#include <linux/uaccess.h>
  28#include <linux/vmalloc.h>
  29#include <linux/ftrace.h>
  30#include <linux/module.h>
  31#include <linux/percpu.h>
  32#include <linux/splice.h>
  33#include <linux/kdebug.h>
  34#include <linux/string.h>
  35#include <linux/mount.h>
  36#include <linux/rwsem.h>
  37#include <linux/slab.h>
  38#include <linux/ctype.h>
  39#include <linux/init.h>
  40#include <linux/poll.h>
  41#include <linux/nmi.h>
  42#include <linux/fs.h>
  43#include <linux/trace.h>
  44#include <linux/sched/clock.h>
  45#include <linux/sched/rt.h>
  46
  47#include "trace.h"
  48#include "trace_output.h"
  49
  50/*
  51 * On boot up, the ring buffer is set to the minimum size, so that
  52 * we do not waste memory on systems that are not using tracing.
  53 */
  54bool ring_buffer_expanded;
  55
  56/*
  57 * We need to change this state when a selftest is running.
  58 * A selftest will lurk into the ring-buffer to count the
  59 * entries inserted during the selftest although some concurrent
  60 * insertions into the ring-buffer such as trace_printk could occurred
  61 * at the same time, giving false positive or negative results.
  62 */
  63static bool __read_mostly tracing_selftest_running;
  64
  65/*
  66 * If a tracer is running, we do not want to run SELFTEST.
  67 */
  68bool __read_mostly tracing_selftest_disabled;
  69
  70/* Pipe tracepoints to printk */
  71struct trace_iterator *tracepoint_print_iter;
  72int tracepoint_printk;
  73static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
  74
  75/* For tracers that don't implement custom flags */
  76static struct tracer_opt dummy_tracer_opt[] = {
  77	{ }
  78};
  79
  80static int
  81dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
  82{
  83	return 0;
  84}
  85
  86/*
  87 * To prevent the comm cache from being overwritten when no
  88 * tracing is active, only save the comm when a trace event
  89 * occurred.
  90 */
  91static DEFINE_PER_CPU(bool, trace_taskinfo_save);
  92
  93/*
  94 * Kill all tracing for good (never come back).
  95 * It is initialized to 1 but will turn to zero if the initialization
  96 * of the tracer is successful. But that is the only place that sets
  97 * this back to zero.
  98 */
  99static int tracing_disabled = 1;
 100
 101cpumask_var_t __read_mostly	tracing_buffer_mask;
 102
 103/*
 104 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
 105 *
 106 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
 107 * is set, then ftrace_dump is called. This will output the contents
 108 * of the ftrace buffers to the console.  This is very useful for
 109 * capturing traces that lead to crashes and outputing it to a
 110 * serial console.
 111 *
 112 * It is default off, but you can enable it with either specifying
 113 * "ftrace_dump_on_oops" in the kernel command line, or setting
 114 * /proc/sys/kernel/ftrace_dump_on_oops
 115 * Set 1 if you want to dump buffers of all CPUs
 116 * Set 2 if you want to dump the buffer of the CPU that triggered oops
 117 */
 118
 119enum ftrace_dump_mode ftrace_dump_on_oops;
 120
 121/* When set, tracing will stop when a WARN*() is hit */
 122int __disable_trace_on_warning;
 123
 124#ifdef CONFIG_TRACE_EVAL_MAP_FILE
 125/* Map of enums to their values, for "eval_map" file */
 126struct trace_eval_map_head {
 127	struct module			*mod;
 128	unsigned long			length;
 129};
 130
 131union trace_eval_map_item;
 132
 133struct trace_eval_map_tail {
 134	/*
 135	 * "end" is first and points to NULL as it must be different
 136	 * than "mod" or "eval_string"
 137	 */
 138	union trace_eval_map_item	*next;
 139	const char			*end;	/* points to NULL */
 140};
 141
 142static DEFINE_MUTEX(trace_eval_mutex);
 143
 144/*
 145 * The trace_eval_maps are saved in an array with two extra elements,
 146 * one at the beginning, and one at the end. The beginning item contains
 147 * the count of the saved maps (head.length), and the module they
 148 * belong to if not built in (head.mod). The ending item contains a
 149 * pointer to the next array of saved eval_map items.
 150 */
 151union trace_eval_map_item {
 152	struct trace_eval_map		map;
 153	struct trace_eval_map_head	head;
 154	struct trace_eval_map_tail	tail;
 155};
 156
 157static union trace_eval_map_item *trace_eval_maps;
 158#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
 159
 160static int tracing_set_tracer(struct trace_array *tr, const char *buf);
 161
 162#define MAX_TRACER_SIZE		100
 163static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
 164static char *default_bootup_tracer;
 165
 166static bool allocate_snapshot;
 167
 168static int __init set_cmdline_ftrace(char *str)
 169{
 170	strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
 171	default_bootup_tracer = bootup_tracer_buf;
 172	/* We are using ftrace early, expand it */
 173	ring_buffer_expanded = true;
 174	return 1;
 175}
 176__setup("ftrace=", set_cmdline_ftrace);
 177
 178static int __init set_ftrace_dump_on_oops(char *str)
 179{
 180	if (*str++ != '=' || !*str) {
 181		ftrace_dump_on_oops = DUMP_ALL;
 182		return 1;
 183	}
 184
 185	if (!strcmp("orig_cpu", str)) {
 186		ftrace_dump_on_oops = DUMP_ORIG;
 187                return 1;
 188        }
 189
 190        return 0;
 191}
 192__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
 193
 194static int __init stop_trace_on_warning(char *str)
 195{
 196	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
 197		__disable_trace_on_warning = 1;
 198	return 1;
 199}
 200__setup("traceoff_on_warning", stop_trace_on_warning);
 201
 202static int __init boot_alloc_snapshot(char *str)
 203{
 204	allocate_snapshot = true;
 205	/* We also need the main ring buffer expanded */
 206	ring_buffer_expanded = true;
 207	return 1;
 208}
 209__setup("alloc_snapshot", boot_alloc_snapshot);
 210
 211
 212static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
 213
 214static int __init set_trace_boot_options(char *str)
 215{
 216	strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
 217	return 0;
 218}
 219__setup("trace_options=", set_trace_boot_options);
 220
 221static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
 222static char *trace_boot_clock __initdata;
 223
 224static int __init set_trace_boot_clock(char *str)
 225{
 226	strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
 227	trace_boot_clock = trace_boot_clock_buf;
 228	return 0;
 229}
 230__setup("trace_clock=", set_trace_boot_clock);
 231
 232static int __init set_tracepoint_printk(char *str)
 233{
 234	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
 235		tracepoint_printk = 1;
 236	return 1;
 237}
 238__setup("tp_printk", set_tracepoint_printk);
 239
 240unsigned long long ns2usecs(u64 nsec)
 241{
 242	nsec += 500;
 243	do_div(nsec, 1000);
 244	return nsec;
 245}
 246
 247/* trace_flags holds trace_options default values */
 248#define TRACE_DEFAULT_FLAGS						\
 249	(FUNCTION_DEFAULT_FLAGS |					\
 250	 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |			\
 251	 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO |		\
 252	 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |			\
 253	 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
 254
 255/* trace_options that are only supported by global_trace */
 256#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK |			\
 257	       TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
 258
 259/* trace_flags that are default zero for instances */
 260#define ZEROED_TRACE_FLAGS \
 261	(TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
 262
 263/*
 264 * The global_trace is the descriptor that holds the top-level tracing
 265 * buffers for the live tracing.
 
 
 
 
 
 
 
 
 266 */
 267static struct trace_array global_trace = {
 268	.trace_flags = TRACE_DEFAULT_FLAGS,
 269};
 270
 271LIST_HEAD(ftrace_trace_arrays);
 272
 273int trace_array_get(struct trace_array *this_tr)
 274{
 275	struct trace_array *tr;
 276	int ret = -ENODEV;
 277
 278	mutex_lock(&trace_types_lock);
 279	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
 280		if (tr == this_tr) {
 281			tr->ref++;
 282			ret = 0;
 283			break;
 284		}
 285	}
 286	mutex_unlock(&trace_types_lock);
 287
 288	return ret;
 289}
 290
 291static void __trace_array_put(struct trace_array *this_tr)
 292{
 293	WARN_ON(!this_tr->ref);
 294	this_tr->ref--;
 295}
 296
 297void trace_array_put(struct trace_array *this_tr)
 298{
 299	mutex_lock(&trace_types_lock);
 300	__trace_array_put(this_tr);
 301	mutex_unlock(&trace_types_lock);
 302}
 303
 304int call_filter_check_discard(struct trace_event_call *call, void *rec,
 305			      struct ring_buffer *buffer,
 306			      struct ring_buffer_event *event)
 307{
 308	if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
 309	    !filter_match_preds(call->filter, rec)) {
 310		__trace_event_discard_commit(buffer, event);
 311		return 1;
 312	}
 313
 314	return 0;
 315}
 
 316
 317void trace_free_pid_list(struct trace_pid_list *pid_list)
 318{
 319	vfree(pid_list->pids);
 320	kfree(pid_list);
 321}
 322
 323/**
 324 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
 325 * @filtered_pids: The list of pids to check
 326 * @search_pid: The PID to find in @filtered_pids
 327 *
 328 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
 329 */
 330bool
 331trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
 332{
 333	/*
 334	 * If pid_max changed after filtered_pids was created, we
 335	 * by default ignore all pids greater than the previous pid_max.
 336	 */
 337	if (search_pid >= filtered_pids->pid_max)
 338		return false;
 339
 340	return test_bit(search_pid, filtered_pids->pids);
 341}
 342
 343/**
 344 * trace_ignore_this_task - should a task be ignored for tracing
 345 * @filtered_pids: The list of pids to check
 346 * @task: The task that should be ignored if not filtered
 347 *
 348 * Checks if @task should be traced or not from @filtered_pids.
 349 * Returns true if @task should *NOT* be traced.
 350 * Returns false if @task should be traced.
 351 */
 352bool
 353trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
 354{
 355	/*
 356	 * Return false, because if filtered_pids does not exist,
 357	 * all pids are good to trace.
 358	 */
 359	if (!filtered_pids)
 360		return false;
 361
 362	return !trace_find_filtered_pid(filtered_pids, task->pid);
 363}
 364
 365/**
 366 * trace_pid_filter_add_remove_task - Add or remove a task from a pid_list
 367 * @pid_list: The list to modify
 368 * @self: The current task for fork or NULL for exit
 369 * @task: The task to add or remove
 370 *
 371 * If adding a task, if @self is defined, the task is only added if @self
 372 * is also included in @pid_list. This happens on fork and tasks should
 373 * only be added when the parent is listed. If @self is NULL, then the
 374 * @task pid will be removed from the list, which would happen on exit
 375 * of a task.
 376 */
 377void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
 378				  struct task_struct *self,
 379				  struct task_struct *task)
 380{
 381	if (!pid_list)
 382		return;
 383
 384	/* For forks, we only add if the forking task is listed */
 385	if (self) {
 386		if (!trace_find_filtered_pid(pid_list, self->pid))
 387			return;
 388	}
 389
 390	/* Sorry, but we don't support pid_max changing after setting */
 391	if (task->pid >= pid_list->pid_max)
 392		return;
 393
 394	/* "self" is set for forks, and NULL for exits */
 395	if (self)
 396		set_bit(task->pid, pid_list->pids);
 397	else
 398		clear_bit(task->pid, pid_list->pids);
 399}
 400
 401/**
 402 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
 403 * @pid_list: The pid list to show
 404 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
 405 * @pos: The position of the file
 406 *
 407 * This is used by the seq_file "next" operation to iterate the pids
 408 * listed in a trace_pid_list structure.
 409 *
 410 * Returns the pid+1 as we want to display pid of zero, but NULL would
 411 * stop the iteration.
 412 */
 413void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
 414{
 415	unsigned long pid = (unsigned long)v;
 416
 417	(*pos)++;
 418
 419	/* pid already is +1 of the actual prevous bit */
 420	pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
 421
 422	/* Return pid + 1 to allow zero to be represented */
 423	if (pid < pid_list->pid_max)
 424		return (void *)(pid + 1);
 425
 426	return NULL;
 427}
 428
 429/**
 430 * trace_pid_start - Used for seq_file to start reading pid lists
 431 * @pid_list: The pid list to show
 432 * @pos: The position of the file
 433 *
 434 * This is used by seq_file "start" operation to start the iteration
 435 * of listing pids.
 436 *
 437 * Returns the pid+1 as we want to display pid of zero, but NULL would
 438 * stop the iteration.
 439 */
 440void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
 441{
 442	unsigned long pid;
 443	loff_t l = 0;
 444
 445	pid = find_first_bit(pid_list->pids, pid_list->pid_max);
 446	if (pid >= pid_list->pid_max)
 447		return NULL;
 448
 449	/* Return pid + 1 so that zero can be the exit value */
 450	for (pid++; pid && l < *pos;
 451	     pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
 452		;
 453	return (void *)pid;
 454}
 455
 456/**
 457 * trace_pid_show - show the current pid in seq_file processing
 458 * @m: The seq_file structure to write into
 459 * @v: A void pointer of the pid (+1) value to display
 460 *
 461 * Can be directly used by seq_file operations to display the current
 462 * pid value.
 463 */
 464int trace_pid_show(struct seq_file *m, void *v)
 465{
 466	unsigned long pid = (unsigned long)v - 1;
 467
 468	seq_printf(m, "%lu\n", pid);
 469	return 0;
 470}
 
 471
 472/* 128 should be much more than enough */
 473#define PID_BUF_SIZE		127
 474
 475int trace_pid_write(struct trace_pid_list *filtered_pids,
 476		    struct trace_pid_list **new_pid_list,
 477		    const char __user *ubuf, size_t cnt)
 478{
 479	struct trace_pid_list *pid_list;
 480	struct trace_parser parser;
 481	unsigned long val;
 482	int nr_pids = 0;
 483	ssize_t read = 0;
 484	ssize_t ret = 0;
 485	loff_t pos;
 486	pid_t pid;
 487
 488	if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
 489		return -ENOMEM;
 490
 491	/*
 492	 * Always recreate a new array. The write is an all or nothing
 493	 * operation. Always create a new array when adding new pids by
 494	 * the user. If the operation fails, then the current list is
 495	 * not modified.
 496	 */
 497	pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
 498	if (!pid_list)
 499		return -ENOMEM;
 500
 501	pid_list->pid_max = READ_ONCE(pid_max);
 502
 503	/* Only truncating will shrink pid_max */
 504	if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
 505		pid_list->pid_max = filtered_pids->pid_max;
 506
 507	pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
 508	if (!pid_list->pids) {
 509		kfree(pid_list);
 510		return -ENOMEM;
 511	}
 512
 513	if (filtered_pids) {
 514		/* copy the current bits to the new max */
 515		for_each_set_bit(pid, filtered_pids->pids,
 516				 filtered_pids->pid_max) {
 517			set_bit(pid, pid_list->pids);
 518			nr_pids++;
 519		}
 520	}
 521
 522	while (cnt > 0) {
 523
 524		pos = 0;
 525
 526		ret = trace_get_user(&parser, ubuf, cnt, &pos);
 527		if (ret < 0 || !trace_parser_loaded(&parser))
 528			break;
 529
 530		read += ret;
 531		ubuf += ret;
 532		cnt -= ret;
 533
 534		ret = -EINVAL;
 535		if (kstrtoul(parser.buffer, 0, &val))
 536			break;
 537		if (val >= pid_list->pid_max)
 538			break;
 539
 540		pid = (pid_t)val;
 541
 542		set_bit(pid, pid_list->pids);
 543		nr_pids++;
 544
 545		trace_parser_clear(&parser);
 546		ret = 0;
 547	}
 548	trace_parser_put(&parser);
 549
 550	if (ret < 0) {
 551		trace_free_pid_list(pid_list);
 552		return ret;
 553	}
 554
 555	if (!nr_pids) {
 556		/* Cleared the list of pids */
 557		trace_free_pid_list(pid_list);
 558		read = ret;
 559		pid_list = NULL;
 560	}
 561
 562	*new_pid_list = pid_list;
 563
 564	return read;
 565}
 566
 567static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu)
 568{
 569	u64 ts;
 570
 571	/* Early boot up does not have a buffer yet */
 572	if (!buf->buffer)
 573		return trace_clock_local();
 574
 575	ts = ring_buffer_time_stamp(buf->buffer, cpu);
 576	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
 577
 578	return ts;
 579}
 580
 581u64 ftrace_now(int cpu)
 582{
 583	return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
 584}
 585
 586/**
 587 * tracing_is_enabled - Show if global_trace has been disabled
 588 *
 589 * Shows if the global trace has been enabled or not. It uses the
 590 * mirror flag "buffer_disabled" to be used in fast paths such as for
 591 * the irqsoff tracer. But it may be inaccurate due to races. If you
 592 * need to know the accurate state, use tracing_is_on() which is a little
 593 * slower, but accurate.
 594 */
 595int tracing_is_enabled(void)
 596{
 597	/*
 598	 * For quick access (irqsoff uses this in fast path), just
 599	 * return the mirror variable of the state of the ring buffer.
 600	 * It's a little racy, but we don't really care.
 601	 */
 602	smp_rmb();
 603	return !global_trace.buffer_disabled;
 604}
 605
 606/*
 607 * trace_buf_size is the size in bytes that is allocated
 608 * for a buffer. Note, the number of bytes is always rounded
 609 * to page size.
 610 *
 611 * This number is purposely set to a low number of 16384.
 612 * If the dump on oops happens, it will be much appreciated
 613 * to not have to wait for all that output. Anyway this can be
 614 * boot time and run time configurable.
 615 */
 616#define TRACE_BUF_SIZE_DEFAULT	1441792UL /* 16384 * 88 (sizeof(entry)) */
 617
 618static unsigned long		trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 619
 620/* trace_types holds a link list of available tracers. */
 621static struct tracer		*trace_types __read_mostly;
 622
 623/*
 624 * trace_types_lock is used to protect the trace_types list.
 625 */
 626DEFINE_MUTEX(trace_types_lock);
 627
 628/*
 629 * serialize the access of the ring buffer
 630 *
 631 * ring buffer serializes readers, but it is low level protection.
 632 * The validity of the events (which returns by ring_buffer_peek() ..etc)
 633 * are not protected by ring buffer.
 634 *
 635 * The content of events may become garbage if we allow other process consumes
 636 * these events concurrently:
 637 *   A) the page of the consumed events may become a normal page
 638 *      (not reader page) in ring buffer, and this page will be rewrited
 639 *      by events producer.
 640 *   B) The page of the consumed events may become a page for splice_read,
 641 *      and this page will be returned to system.
 642 *
 643 * These primitives allow multi process access to different cpu ring buffer
 644 * concurrently.
 645 *
 646 * These primitives don't distinguish read-only and read-consume access.
 647 * Multi read-only access are also serialized.
 648 */
 649
 650#ifdef CONFIG_SMP
 651static DECLARE_RWSEM(all_cpu_access_lock);
 652static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
 653
 654static inline void trace_access_lock(int cpu)
 655{
 656	if (cpu == RING_BUFFER_ALL_CPUS) {
 657		/* gain it for accessing the whole ring buffer. */
 658		down_write(&all_cpu_access_lock);
 659	} else {
 660		/* gain it for accessing a cpu ring buffer. */
 661
 662		/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
 663		down_read(&all_cpu_access_lock);
 664
 665		/* Secondly block other access to this @cpu ring buffer. */
 666		mutex_lock(&per_cpu(cpu_access_lock, cpu));
 667	}
 668}
 669
 670static inline void trace_access_unlock(int cpu)
 671{
 672	if (cpu == RING_BUFFER_ALL_CPUS) {
 673		up_write(&all_cpu_access_lock);
 674	} else {
 675		mutex_unlock(&per_cpu(cpu_access_lock, cpu));
 676		up_read(&all_cpu_access_lock);
 677	}
 678}
 679
 680static inline void trace_access_lock_init(void)
 681{
 682	int cpu;
 683
 684	for_each_possible_cpu(cpu)
 685		mutex_init(&per_cpu(cpu_access_lock, cpu));
 686}
 687
 688#else
 689
 690static DEFINE_MUTEX(access_lock);
 691
 692static inline void trace_access_lock(int cpu)
 693{
 694	(void)cpu;
 695	mutex_lock(&access_lock);
 696}
 697
 698static inline void trace_access_unlock(int cpu)
 699{
 700	(void)cpu;
 701	mutex_unlock(&access_lock);
 702}
 703
 704static inline void trace_access_lock_init(void)
 705{
 706}
 707
 708#endif
 709
 710#ifdef CONFIG_STACKTRACE
 711static void __ftrace_trace_stack(struct ring_buffer *buffer,
 712				 unsigned long flags,
 713				 int skip, int pc, struct pt_regs *regs);
 714static inline void ftrace_trace_stack(struct trace_array *tr,
 715				      struct ring_buffer *buffer,
 716				      unsigned long flags,
 717				      int skip, int pc, struct pt_regs *regs);
 718
 719#else
 720static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
 721					unsigned long flags,
 722					int skip, int pc, struct pt_regs *regs)
 723{
 724}
 725static inline void ftrace_trace_stack(struct trace_array *tr,
 726				      struct ring_buffer *buffer,
 727				      unsigned long flags,
 728				      int skip, int pc, struct pt_regs *regs)
 729{
 730}
 731
 732#endif
 733
 734static __always_inline void
 735trace_event_setup(struct ring_buffer_event *event,
 736		  int type, unsigned long flags, int pc)
 737{
 738	struct trace_entry *ent = ring_buffer_event_data(event);
 739
 740	tracing_generic_entry_update(ent, flags, pc);
 741	ent->type = type;
 742}
 743
 744static __always_inline struct ring_buffer_event *
 745__trace_buffer_lock_reserve(struct ring_buffer *buffer,
 746			  int type,
 747			  unsigned long len,
 748			  unsigned long flags, int pc)
 749{
 750	struct ring_buffer_event *event;
 751
 752	event = ring_buffer_lock_reserve(buffer, len);
 753	if (event != NULL)
 754		trace_event_setup(event, type, flags, pc);
 755
 756	return event;
 757}
 758
 759void tracer_tracing_on(struct trace_array *tr)
 760{
 761	if (tr->trace_buffer.buffer)
 762		ring_buffer_record_on(tr->trace_buffer.buffer);
 763	/*
 764	 * This flag is looked at when buffers haven't been allocated
 765	 * yet, or by some tracers (like irqsoff), that just want to
 766	 * know if the ring buffer has been disabled, but it can handle
 767	 * races of where it gets disabled but we still do a record.
 768	 * As the check is in the fast path of the tracers, it is more
 769	 * important to be fast than accurate.
 770	 */
 771	tr->buffer_disabled = 0;
 772	/* Make the flag seen by readers */
 773	smp_wmb();
 774}
 775
 776/**
 777 * tracing_on - enable tracing buffers
 778 *
 779 * This function enables tracing buffers that may have been
 780 * disabled with tracing_off.
 781 */
 782void tracing_on(void)
 783{
 784	tracer_tracing_on(&global_trace);
 785}
 786EXPORT_SYMBOL_GPL(tracing_on);
 787
 788
 789static __always_inline void
 790__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
 791{
 792	__this_cpu_write(trace_taskinfo_save, true);
 793
 794	/* If this is the temp buffer, we need to commit fully */
 795	if (this_cpu_read(trace_buffered_event) == event) {
 796		/* Length is in event->array[0] */
 797		ring_buffer_write(buffer, event->array[0], &event->array[1]);
 798		/* Release the temp buffer */
 799		this_cpu_dec(trace_buffered_event_cnt);
 800	} else
 801		ring_buffer_unlock_commit(buffer, event);
 802}
 803
 804/**
 805 * __trace_puts - write a constant string into the trace buffer.
 806 * @ip:	   The address of the caller
 807 * @str:   The constant string to write
 808 * @size:  The size of the string.
 809 */
 810int __trace_puts(unsigned long ip, const char *str, int size)
 811{
 812	struct ring_buffer_event *event;
 813	struct ring_buffer *buffer;
 814	struct print_entry *entry;
 815	unsigned long irq_flags;
 816	int alloc;
 817	int pc;
 818
 819	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
 820		return 0;
 821
 822	pc = preempt_count();
 823
 824	if (unlikely(tracing_selftest_running || tracing_disabled))
 825		return 0;
 826
 827	alloc = sizeof(*entry) + size + 2; /* possible \n added */
 828
 829	local_save_flags(irq_flags);
 830	buffer = global_trace.trace_buffer.buffer;
 831	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
 832					    irq_flags, pc);
 833	if (!event)
 834		return 0;
 835
 836	entry = ring_buffer_event_data(event);
 837	entry->ip = ip;
 838
 839	memcpy(&entry->buf, str, size);
 840
 841	/* Add a newline if necessary */
 842	if (entry->buf[size - 1] != '\n') {
 843		entry->buf[size] = '\n';
 844		entry->buf[size + 1] = '\0';
 845	} else
 846		entry->buf[size] = '\0';
 847
 848	__buffer_unlock_commit(buffer, event);
 849	ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
 850
 851	return size;
 852}
 853EXPORT_SYMBOL_GPL(__trace_puts);
 854
 855/**
 856 * __trace_bputs - write the pointer to a constant string into trace buffer
 857 * @ip:	   The address of the caller
 858 * @str:   The constant string to write to the buffer to
 859 */
 860int __trace_bputs(unsigned long ip, const char *str)
 861{
 862	struct ring_buffer_event *event;
 863	struct ring_buffer *buffer;
 864	struct bputs_entry *entry;
 865	unsigned long irq_flags;
 866	int size = sizeof(struct bputs_entry);
 867	int pc;
 868
 869	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
 870		return 0;
 871
 872	pc = preempt_count();
 873
 874	if (unlikely(tracing_selftest_running || tracing_disabled))
 875		return 0;
 876
 877	local_save_flags(irq_flags);
 878	buffer = global_trace.trace_buffer.buffer;
 879	event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
 880					    irq_flags, pc);
 881	if (!event)
 882		return 0;
 883
 884	entry = ring_buffer_event_data(event);
 885	entry->ip			= ip;
 886	entry->str			= str;
 887
 888	__buffer_unlock_commit(buffer, event);
 889	ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
 890
 891	return 1;
 892}
 893EXPORT_SYMBOL_GPL(__trace_bputs);
 894
 895#ifdef CONFIG_TRACER_SNAPSHOT
 896void tracing_snapshot_instance(struct trace_array *tr)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 897{
 
 898	struct tracer *tracer = tr->current_trace;
 899	unsigned long flags;
 900
 901	if (in_nmi()) {
 902		internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
 903		internal_trace_puts("*** snapshot is being ignored        ***\n");
 904		return;
 905	}
 906
 907	if (!tr->allocated_snapshot) {
 908		internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
 909		internal_trace_puts("*** stopping trace here!   ***\n");
 910		tracing_off();
 911		return;
 912	}
 913
 914	/* Note, snapshot can not be used when the tracer uses it */
 915	if (tracer->use_max_tr) {
 916		internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
 917		internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
 918		return;
 919	}
 920
 921	local_irq_save(flags);
 922	update_max_tr(tr, current, smp_processor_id());
 923	local_irq_restore(flags);
 924}
 925
 926/**
 927 * tracing_snapshot - take a snapshot of the current buffer.
 928 *
 929 * This causes a swap between the snapshot buffer and the current live
 930 * tracing buffer. You can use this to take snapshots of the live
 931 * trace when some condition is triggered, but continue to trace.
 932 *
 933 * Note, make sure to allocate the snapshot with either
 934 * a tracing_snapshot_alloc(), or by doing it manually
 935 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
 936 *
 937 * If the snapshot buffer is not allocated, it will stop tracing.
 938 * Basically making a permanent snapshot.
 939 */
 940void tracing_snapshot(void)
 941{
 942	struct trace_array *tr = &global_trace;
 943
 944	tracing_snapshot_instance(tr);
 945}
 946EXPORT_SYMBOL_GPL(tracing_snapshot);
 947
 948static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
 949					struct trace_buffer *size_buf, int cpu_id);
 950static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
 951
 952int tracing_alloc_snapshot_instance(struct trace_array *tr)
 953{
 954	int ret;
 955
 956	if (!tr->allocated_snapshot) {
 957
 958		/* allocate spare buffer */
 959		ret = resize_buffer_duplicate_size(&tr->max_buffer,
 960				   &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
 961		if (ret < 0)
 962			return ret;
 963
 964		tr->allocated_snapshot = true;
 965	}
 966
 967	return 0;
 968}
 969
 970static void free_snapshot(struct trace_array *tr)
 971{
 972	/*
 973	 * We don't free the ring buffer. instead, resize it because
 974	 * The max_tr ring buffer has some state (e.g. ring->clock) and
 975	 * we want preserve it.
 976	 */
 977	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
 978	set_buffer_entries(&tr->max_buffer, 1);
 979	tracing_reset_online_cpus(&tr->max_buffer);
 980	tr->allocated_snapshot = false;
 981}
 982
 983/**
 984 * tracing_alloc_snapshot - allocate snapshot buffer.
 985 *
 986 * This only allocates the snapshot buffer if it isn't already
 987 * allocated - it doesn't also take a snapshot.
 988 *
 989 * This is meant to be used in cases where the snapshot buffer needs
 990 * to be set up for events that can't sleep but need to be able to
 991 * trigger a snapshot.
 992 */
 993int tracing_alloc_snapshot(void)
 994{
 995	struct trace_array *tr = &global_trace;
 996	int ret;
 997
 998	ret = tracing_alloc_snapshot_instance(tr);
 999	WARN_ON(ret < 0);
1000
1001	return ret;
1002}
1003EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1004
1005/**
1006 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1007 *
1008 * This is similar to tracing_snapshot(), but it will allocate the
1009 * snapshot buffer if it isn't already allocated. Use this only
1010 * where it is safe to sleep, as the allocation may sleep.
1011 *
1012 * This causes a swap between the snapshot buffer and the current live
1013 * tracing buffer. You can use this to take snapshots of the live
1014 * trace when some condition is triggered, but continue to trace.
1015 */
1016void tracing_snapshot_alloc(void)
1017{
1018	int ret;
1019
1020	ret = tracing_alloc_snapshot();
1021	if (ret < 0)
1022		return;
1023
1024	tracing_snapshot();
1025}
1026EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1027#else
1028void tracing_snapshot(void)
1029{
1030	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1031}
1032EXPORT_SYMBOL_GPL(tracing_snapshot);
1033int tracing_alloc_snapshot(void)
1034{
1035	WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1036	return -ENODEV;
1037}
1038EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1039void tracing_snapshot_alloc(void)
1040{
1041	/* Give warning */
1042	tracing_snapshot();
1043}
1044EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1045#endif /* CONFIG_TRACER_SNAPSHOT */
1046
1047void tracer_tracing_off(struct trace_array *tr)
1048{
1049	if (tr->trace_buffer.buffer)
1050		ring_buffer_record_off(tr->trace_buffer.buffer);
1051	/*
1052	 * This flag is looked at when buffers haven't been allocated
1053	 * yet, or by some tracers (like irqsoff), that just want to
1054	 * know if the ring buffer has been disabled, but it can handle
1055	 * races of where it gets disabled but we still do a record.
1056	 * As the check is in the fast path of the tracers, it is more
1057	 * important to be fast than accurate.
1058	 */
1059	tr->buffer_disabled = 1;
1060	/* Make the flag seen by readers */
1061	smp_wmb();
1062}
1063
1064/**
1065 * tracing_off - turn off tracing buffers
1066 *
1067 * This function stops the tracing buffers from recording data.
1068 * It does not disable any overhead the tracers themselves may
1069 * be causing. This function simply causes all recording to
1070 * the ring buffers to fail.
1071 */
1072void tracing_off(void)
1073{
1074	tracer_tracing_off(&global_trace);
1075}
1076EXPORT_SYMBOL_GPL(tracing_off);
1077
1078void disable_trace_on_warning(void)
1079{
1080	if (__disable_trace_on_warning)
1081		tracing_off();
1082}
1083
1084/**
1085 * tracer_tracing_is_on - show real state of ring buffer enabled
1086 * @tr : the trace array to know if ring buffer is enabled
1087 *
1088 * Shows real state of the ring buffer if it is enabled or not.
1089 */
1090int tracer_tracing_is_on(struct trace_array *tr)
1091{
1092	if (tr->trace_buffer.buffer)
1093		return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1094	return !tr->buffer_disabled;
1095}
1096
1097/**
1098 * tracing_is_on - show state of ring buffers enabled
1099 */
1100int tracing_is_on(void)
1101{
1102	return tracer_tracing_is_on(&global_trace);
1103}
1104EXPORT_SYMBOL_GPL(tracing_is_on);
1105
1106static int __init set_buf_size(char *str)
1107{
1108	unsigned long buf_size;
1109
1110	if (!str)
1111		return 0;
1112	buf_size = memparse(str, &str);
1113	/* nr_entries can not be zero */
1114	if (buf_size == 0)
1115		return 0;
1116	trace_buf_size = buf_size;
1117	return 1;
1118}
1119__setup("trace_buf_size=", set_buf_size);
1120
1121static int __init set_tracing_thresh(char *str)
1122{
1123	unsigned long threshold;
1124	int ret;
1125
1126	if (!str)
1127		return 0;
1128	ret = kstrtoul(str, 0, &threshold);
1129	if (ret < 0)
1130		return 0;
1131	tracing_thresh = threshold * 1000;
1132	return 1;
1133}
1134__setup("tracing_thresh=", set_tracing_thresh);
1135
1136unsigned long nsecs_to_usecs(unsigned long nsecs)
1137{
1138	return nsecs / 1000;
1139}
1140
1141/*
1142 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1143 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1144 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1145 * of strings in the order that the evals (enum) were defined.
1146 */
1147#undef C
1148#define C(a, b) b
1149
1150/* These must match the bit postions in trace_iterator_flags */
1151static const char *trace_options[] = {
1152	TRACE_FLAGS
1153	NULL
1154};
1155
1156static struct {
1157	u64 (*func)(void);
1158	const char *name;
1159	int in_ns;		/* is this clock in nanoseconds? */
1160} trace_clocks[] = {
1161	{ trace_clock_local,		"local",	1 },
1162	{ trace_clock_global,		"global",	1 },
1163	{ trace_clock_counter,		"counter",	0 },
1164	{ trace_clock_jiffies,		"uptime",	0 },
1165	{ trace_clock,			"perf",		1 },
1166	{ ktime_get_mono_fast_ns,	"mono",		1 },
1167	{ ktime_get_raw_fast_ns,	"mono_raw",	1 },
1168	{ ktime_get_boot_fast_ns,	"boot",		1 },
1169	ARCH_TRACE_CLOCKS
1170};
1171
1172bool trace_clock_in_ns(struct trace_array *tr)
1173{
1174	if (trace_clocks[tr->clock_id].in_ns)
1175		return true;
1176
1177	return false;
1178}
1179
1180/*
1181 * trace_parser_get_init - gets the buffer for trace parser
1182 */
1183int trace_parser_get_init(struct trace_parser *parser, int size)
1184{
1185	memset(parser, 0, sizeof(*parser));
1186
1187	parser->buffer = kmalloc(size, GFP_KERNEL);
1188	if (!parser->buffer)
1189		return 1;
1190
1191	parser->size = size;
1192	return 0;
1193}
1194
1195/*
1196 * trace_parser_put - frees the buffer for trace parser
1197 */
1198void trace_parser_put(struct trace_parser *parser)
1199{
1200	kfree(parser->buffer);
1201	parser->buffer = NULL;
1202}
1203
1204/*
1205 * trace_get_user - reads the user input string separated by  space
1206 * (matched by isspace(ch))
1207 *
1208 * For each string found the 'struct trace_parser' is updated,
1209 * and the function returns.
1210 *
1211 * Returns number of bytes read.
1212 *
1213 * See kernel/trace/trace.h for 'struct trace_parser' details.
1214 */
1215int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1216	size_t cnt, loff_t *ppos)
1217{
1218	char ch;
1219	size_t read = 0;
1220	ssize_t ret;
1221
1222	if (!*ppos)
1223		trace_parser_clear(parser);
1224
1225	ret = get_user(ch, ubuf++);
1226	if (ret)
1227		goto out;
1228
1229	read++;
1230	cnt--;
1231
1232	/*
1233	 * The parser is not finished with the last write,
1234	 * continue reading the user input without skipping spaces.
1235	 */
1236	if (!parser->cont) {
1237		/* skip white space */
1238		while (cnt && isspace(ch)) {
1239			ret = get_user(ch, ubuf++);
1240			if (ret)
1241				goto out;
1242			read++;
1243			cnt--;
1244		}
1245
1246		parser->idx = 0;
1247
1248		/* only spaces were written */
1249		if (isspace(ch) || !ch) {
1250			*ppos += read;
1251			ret = read;
1252			goto out;
1253		}
 
 
1254	}
1255
1256	/* read the non-space input */
1257	while (cnt && !isspace(ch) && ch) {
1258		if (parser->idx < parser->size - 1)
1259			parser->buffer[parser->idx++] = ch;
1260		else {
1261			ret = -EINVAL;
1262			goto out;
1263		}
1264		ret = get_user(ch, ubuf++);
1265		if (ret)
1266			goto out;
1267		read++;
1268		cnt--;
1269	}
1270
1271	/* We either got finished input or we have to wait for another call. */
1272	if (isspace(ch) || !ch) {
1273		parser->buffer[parser->idx] = 0;
1274		parser->cont = false;
1275	} else if (parser->idx < parser->size - 1) {
1276		parser->cont = true;
1277		parser->buffer[parser->idx++] = ch;
1278		/* Make sure the parsed string always terminates with '\0'. */
1279		parser->buffer[parser->idx] = 0;
1280	} else {
1281		ret = -EINVAL;
1282		goto out;
1283	}
1284
1285	*ppos += read;
1286	ret = read;
1287
1288out:
1289	return ret;
1290}
1291
1292/* TODO add a seq_buf_to_buffer() */
1293static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1294{
1295	int len;
1296
1297	if (trace_seq_used(s) <= s->seq.readpos)
1298		return -EBUSY;
1299
1300	len = trace_seq_used(s) - s->seq.readpos;
1301	if (cnt > len)
1302		cnt = len;
1303	memcpy(buf, s->buffer + s->seq.readpos, cnt);
1304
1305	s->seq.readpos += cnt;
1306	return cnt;
1307}
1308
1309unsigned long __read_mostly	tracing_thresh;
1310
1311#ifdef CONFIG_TRACER_MAX_TRACE
1312/*
1313 * Copy the new maximum trace into the separate maximum-trace
1314 * structure. (this way the maximum trace is permanently saved,
1315 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1316 */
1317static void
1318__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1319{
1320	struct trace_buffer *trace_buf = &tr->trace_buffer;
1321	struct trace_buffer *max_buf = &tr->max_buffer;
1322	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1323	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1324
1325	max_buf->cpu = cpu;
1326	max_buf->time_start = data->preempt_timestamp;
1327
1328	max_data->saved_latency = tr->max_latency;
1329	max_data->critical_start = data->critical_start;
1330	max_data->critical_end = data->critical_end;
1331
1332	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1333	max_data->pid = tsk->pid;
1334	/*
1335	 * If tsk == current, then use current_uid(), as that does not use
1336	 * RCU. The irq tracer can be called out of RCU scope.
1337	 */
1338	if (tsk == current)
1339		max_data->uid = current_uid();
1340	else
1341		max_data->uid = task_uid(tsk);
1342
1343	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1344	max_data->policy = tsk->policy;
1345	max_data->rt_priority = tsk->rt_priority;
1346
1347	/* record this tasks comm */
1348	tracing_record_cmdline(tsk);
1349}
1350
1351/**
1352 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1353 * @tr: tracer
1354 * @tsk: the task with the latency
1355 * @cpu: The cpu that initiated the trace.
1356 *
1357 * Flip the buffers between the @tr and the max_tr and record information
1358 * about which task was the cause of this latency.
1359 */
1360void
1361update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1362{
1363	struct ring_buffer *buf;
1364
1365	if (tr->stop_count)
1366		return;
1367
1368	WARN_ON_ONCE(!irqs_disabled());
1369
1370	if (!tr->allocated_snapshot) {
1371		/* Only the nop tracer should hit this when disabling */
1372		WARN_ON_ONCE(tr->current_trace != &nop_trace);
1373		return;
1374	}
1375
1376	arch_spin_lock(&tr->max_lock);
1377
1378	buf = tr->trace_buffer.buffer;
1379	tr->trace_buffer.buffer = tr->max_buffer.buffer;
1380	tr->max_buffer.buffer = buf;
1381
1382	__update_max_tr(tr, tsk, cpu);
1383	arch_spin_unlock(&tr->max_lock);
1384}
1385
1386/**
1387 * update_max_tr_single - only copy one trace over, and reset the rest
1388 * @tr - tracer
1389 * @tsk - task with the latency
1390 * @cpu - the cpu of the buffer to copy.
1391 *
1392 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1393 */
1394void
1395update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1396{
1397	int ret;
1398
1399	if (tr->stop_count)
1400		return;
1401
1402	WARN_ON_ONCE(!irqs_disabled());
1403	if (!tr->allocated_snapshot) {
1404		/* Only the nop tracer should hit this when disabling */
1405		WARN_ON_ONCE(tr->current_trace != &nop_trace);
1406		return;
1407	}
1408
1409	arch_spin_lock(&tr->max_lock);
1410
1411	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1412
1413	if (ret == -EBUSY) {
1414		/*
1415		 * We failed to swap the buffer due to a commit taking
1416		 * place on this CPU. We fail to record, but we reset
1417		 * the max trace buffer (no one writes directly to it)
1418		 * and flag that it failed.
1419		 */
1420		trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1421			"Failed to swap buffers due to commit in progress\n");
1422	}
1423
1424	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1425
1426	__update_max_tr(tr, tsk, cpu);
1427	arch_spin_unlock(&tr->max_lock);
1428}
1429#endif /* CONFIG_TRACER_MAX_TRACE */
1430
1431static int wait_on_pipe(struct trace_iterator *iter, bool full)
1432{
1433	/* Iterators are static, they should be filled or empty */
1434	if (trace_buffer_iter(iter, iter->cpu_file))
1435		return 0;
1436
1437	return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1438				full);
1439}
1440
1441#ifdef CONFIG_FTRACE_STARTUP_TEST
1442static bool selftests_can_run;
1443
1444struct trace_selftests {
1445	struct list_head		list;
1446	struct tracer			*type;
1447};
1448
1449static LIST_HEAD(postponed_selftests);
1450
1451static int save_selftest(struct tracer *type)
1452{
1453	struct trace_selftests *selftest;
1454
1455	selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1456	if (!selftest)
1457		return -ENOMEM;
1458
1459	selftest->type = type;
1460	list_add(&selftest->list, &postponed_selftests);
1461	return 0;
1462}
1463
1464static int run_tracer_selftest(struct tracer *type)
1465{
1466	struct trace_array *tr = &global_trace;
1467	struct tracer *saved_tracer = tr->current_trace;
1468	int ret;
1469
1470	if (!type->selftest || tracing_selftest_disabled)
1471		return 0;
1472
1473	/*
1474	 * If a tracer registers early in boot up (before scheduling is
1475	 * initialized and such), then do not run its selftests yet.
1476	 * Instead, run it a little later in the boot process.
1477	 */
1478	if (!selftests_can_run)
1479		return save_selftest(type);
1480
1481	/*
1482	 * Run a selftest on this tracer.
1483	 * Here we reset the trace buffer, and set the current
1484	 * tracer to be this tracer. The tracer can then run some
1485	 * internal tracing to verify that everything is in order.
1486	 * If we fail, we do not register this tracer.
1487	 */
1488	tracing_reset_online_cpus(&tr->trace_buffer);
1489
1490	tr->current_trace = type;
1491
1492#ifdef CONFIG_TRACER_MAX_TRACE
1493	if (type->use_max_tr) {
1494		/* If we expanded the buffers, make sure the max is expanded too */
1495		if (ring_buffer_expanded)
1496			ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1497					   RING_BUFFER_ALL_CPUS);
1498		tr->allocated_snapshot = true;
1499	}
1500#endif
1501
1502	/* the test is responsible for initializing and enabling */
1503	pr_info("Testing tracer %s: ", type->name);
1504	ret = type->selftest(type, tr);
1505	/* the test is responsible for resetting too */
1506	tr->current_trace = saved_tracer;
1507	if (ret) {
1508		printk(KERN_CONT "FAILED!\n");
1509		/* Add the warning after printing 'FAILED' */
1510		WARN_ON(1);
1511		return -1;
1512	}
1513	/* Only reset on passing, to avoid touching corrupted buffers */
1514	tracing_reset_online_cpus(&tr->trace_buffer);
1515
1516#ifdef CONFIG_TRACER_MAX_TRACE
1517	if (type->use_max_tr) {
1518		tr->allocated_snapshot = false;
1519
1520		/* Shrink the max buffer again */
1521		if (ring_buffer_expanded)
1522			ring_buffer_resize(tr->max_buffer.buffer, 1,
1523					   RING_BUFFER_ALL_CPUS);
1524	}
1525#endif
1526
1527	printk(KERN_CONT "PASSED\n");
1528	return 0;
1529}
1530
1531static __init int init_trace_selftests(void)
1532{
1533	struct trace_selftests *p, *n;
1534	struct tracer *t, **last;
1535	int ret;
1536
1537	selftests_can_run = true;
1538
1539	mutex_lock(&trace_types_lock);
1540
1541	if (list_empty(&postponed_selftests))
1542		goto out;
1543
1544	pr_info("Running postponed tracer tests:\n");
1545
1546	list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1547		ret = run_tracer_selftest(p->type);
1548		/* If the test fails, then warn and remove from available_tracers */
1549		if (ret < 0) {
1550			WARN(1, "tracer: %s failed selftest, disabling\n",
1551			     p->type->name);
1552			last = &trace_types;
1553			for (t = trace_types; t; t = t->next) {
1554				if (t == p->type) {
1555					*last = t->next;
1556					break;
1557				}
1558				last = &t->next;
1559			}
1560		}
1561		list_del(&p->list);
1562		kfree(p);
1563	}
1564
1565 out:
1566	mutex_unlock(&trace_types_lock);
1567
1568	return 0;
1569}
1570core_initcall(init_trace_selftests);
1571#else
1572static inline int run_tracer_selftest(struct tracer *type)
1573{
1574	return 0;
1575}
1576#endif /* CONFIG_FTRACE_STARTUP_TEST */
1577
1578static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1579
1580static void __init apply_trace_boot_options(void);
1581
1582/**
1583 * register_tracer - register a tracer with the ftrace system.
1584 * @type - the plugin for the tracer
1585 *
1586 * Register a new plugin tracer.
1587 */
1588int __init register_tracer(struct tracer *type)
1589{
1590	struct tracer *t;
1591	int ret = 0;
1592
1593	if (!type->name) {
1594		pr_info("Tracer must have a name\n");
1595		return -1;
1596	}
1597
1598	if (strlen(type->name) >= MAX_TRACER_SIZE) {
1599		pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1600		return -1;
1601	}
1602
1603	mutex_lock(&trace_types_lock);
1604
1605	tracing_selftest_running = true;
1606
1607	for (t = trace_types; t; t = t->next) {
1608		if (strcmp(type->name, t->name) == 0) {
1609			/* already found */
1610			pr_info("Tracer %s already registered\n",
1611				type->name);
1612			ret = -1;
1613			goto out;
1614		}
1615	}
1616
1617	if (!type->set_flag)
1618		type->set_flag = &dummy_set_flag;
1619	if (!type->flags) {
1620		/*allocate a dummy tracer_flags*/
1621		type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1622		if (!type->flags) {
1623			ret = -ENOMEM;
1624			goto out;
1625		}
1626		type->flags->val = 0;
1627		type->flags->opts = dummy_tracer_opt;
1628	} else
1629		if (!type->flags->opts)
1630			type->flags->opts = dummy_tracer_opt;
1631
1632	/* store the tracer for __set_tracer_option */
1633	type->flags->trace = type;
1634
1635	ret = run_tracer_selftest(type);
1636	if (ret < 0)
1637		goto out;
1638
1639	type->next = trace_types;
1640	trace_types = type;
1641	add_tracer_options(&global_trace, type);
1642
1643 out:
1644	tracing_selftest_running = false;
1645	mutex_unlock(&trace_types_lock);
1646
1647	if (ret || !default_bootup_tracer)
1648		goto out_unlock;
1649
1650	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1651		goto out_unlock;
1652
1653	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1654	/* Do we want this tracer to start on bootup? */
1655	tracing_set_tracer(&global_trace, type->name);
1656	default_bootup_tracer = NULL;
1657
1658	apply_trace_boot_options();
1659
1660	/* disable other selftests, since this will break it. */
1661	tracing_selftest_disabled = true;
1662#ifdef CONFIG_FTRACE_STARTUP_TEST
1663	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1664	       type->name);
1665#endif
1666
1667 out_unlock:
1668	return ret;
1669}
1670
1671void tracing_reset(struct trace_buffer *buf, int cpu)
1672{
1673	struct ring_buffer *buffer = buf->buffer;
1674
1675	if (!buffer)
1676		return;
1677
1678	ring_buffer_record_disable(buffer);
1679
1680	/* Make sure all commits have finished */
1681	synchronize_sched();
1682	ring_buffer_reset_cpu(buffer, cpu);
1683
1684	ring_buffer_record_enable(buffer);
1685}
1686
1687void tracing_reset_online_cpus(struct trace_buffer *buf)
1688{
1689	struct ring_buffer *buffer = buf->buffer;
1690	int cpu;
1691
1692	if (!buffer)
1693		return;
1694
1695	ring_buffer_record_disable(buffer);
1696
1697	/* Make sure all commits have finished */
1698	synchronize_sched();
1699
1700	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1701
1702	for_each_online_cpu(cpu)
1703		ring_buffer_reset_cpu(buffer, cpu);
1704
1705	ring_buffer_record_enable(buffer);
1706}
1707
1708/* Must have trace_types_lock held */
1709void tracing_reset_all_online_cpus(void)
1710{
1711	struct trace_array *tr;
1712
1713	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1714		if (!tr->clear_trace)
1715			continue;
1716		tr->clear_trace = false;
1717		tracing_reset_online_cpus(&tr->trace_buffer);
1718#ifdef CONFIG_TRACER_MAX_TRACE
1719		tracing_reset_online_cpus(&tr->max_buffer);
1720#endif
1721	}
1722}
1723
1724static int *tgid_map;
1725
1726#define SAVED_CMDLINES_DEFAULT 128
1727#define NO_CMDLINE_MAP UINT_MAX
1728static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1729struct saved_cmdlines_buffer {
1730	unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1731	unsigned *map_cmdline_to_pid;
1732	unsigned cmdline_num;
1733	int cmdline_idx;
1734	char *saved_cmdlines;
1735};
1736static struct saved_cmdlines_buffer *savedcmd;
1737
1738/* temporary disable recording */
1739static atomic_t trace_record_taskinfo_disabled __read_mostly;
1740
1741static inline char *get_saved_cmdlines(int idx)
1742{
1743	return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1744}
1745
1746static inline void set_cmdline(int idx, const char *cmdline)
1747{
1748	memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1749}
1750
1751static int allocate_cmdlines_buffer(unsigned int val,
1752				    struct saved_cmdlines_buffer *s)
1753{
1754	s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1755					GFP_KERNEL);
1756	if (!s->map_cmdline_to_pid)
1757		return -ENOMEM;
1758
1759	s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1760	if (!s->saved_cmdlines) {
1761		kfree(s->map_cmdline_to_pid);
1762		return -ENOMEM;
1763	}
1764
1765	s->cmdline_idx = 0;
1766	s->cmdline_num = val;
1767	memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1768	       sizeof(s->map_pid_to_cmdline));
1769	memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1770	       val * sizeof(*s->map_cmdline_to_pid));
1771
1772	return 0;
1773}
1774
1775static int trace_create_savedcmd(void)
1776{
1777	int ret;
1778
1779	savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1780	if (!savedcmd)
1781		return -ENOMEM;
1782
1783	ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1784	if (ret < 0) {
1785		kfree(savedcmd);
1786		savedcmd = NULL;
1787		return -ENOMEM;
1788	}
1789
1790	return 0;
1791}
1792
1793int is_tracing_stopped(void)
1794{
1795	return global_trace.stop_count;
1796}
1797
1798/**
1799 * tracing_start - quick start of the tracer
1800 *
1801 * If tracing is enabled but was stopped by tracing_stop,
1802 * this will start the tracer back up.
1803 */
1804void tracing_start(void)
1805{
1806	struct ring_buffer *buffer;
1807	unsigned long flags;
1808
1809	if (tracing_disabled)
1810		return;
1811
1812	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1813	if (--global_trace.stop_count) {
1814		if (global_trace.stop_count < 0) {
1815			/* Someone screwed up their debugging */
1816			WARN_ON_ONCE(1);
1817			global_trace.stop_count = 0;
1818		}
1819		goto out;
1820	}
1821
1822	/* Prevent the buffers from switching */
1823	arch_spin_lock(&global_trace.max_lock);
1824
1825	buffer = global_trace.trace_buffer.buffer;
1826	if (buffer)
1827		ring_buffer_record_enable(buffer);
1828
1829#ifdef CONFIG_TRACER_MAX_TRACE
1830	buffer = global_trace.max_buffer.buffer;
1831	if (buffer)
1832		ring_buffer_record_enable(buffer);
1833#endif
1834
1835	arch_spin_unlock(&global_trace.max_lock);
1836
1837 out:
1838	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1839}
1840
1841static void tracing_start_tr(struct trace_array *tr)
1842{
1843	struct ring_buffer *buffer;
1844	unsigned long flags;
1845
1846	if (tracing_disabled)
1847		return;
1848
1849	/* If global, we need to also start the max tracer */
1850	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1851		return tracing_start();
1852
1853	raw_spin_lock_irqsave(&tr->start_lock, flags);
1854
1855	if (--tr->stop_count) {
1856		if (tr->stop_count < 0) {
1857			/* Someone screwed up their debugging */
1858			WARN_ON_ONCE(1);
1859			tr->stop_count = 0;
1860		}
1861		goto out;
1862	}
1863
1864	buffer = tr->trace_buffer.buffer;
1865	if (buffer)
1866		ring_buffer_record_enable(buffer);
1867
1868 out:
1869	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1870}
1871
1872/**
1873 * tracing_stop - quick stop of the tracer
1874 *
1875 * Light weight way to stop tracing. Use in conjunction with
1876 * tracing_start.
1877 */
1878void tracing_stop(void)
1879{
1880	struct ring_buffer *buffer;
1881	unsigned long flags;
1882
1883	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1884	if (global_trace.stop_count++)
1885		goto out;
1886
1887	/* Prevent the buffers from switching */
1888	arch_spin_lock(&global_trace.max_lock);
1889
1890	buffer = global_trace.trace_buffer.buffer;
1891	if (buffer)
1892		ring_buffer_record_disable(buffer);
1893
1894#ifdef CONFIG_TRACER_MAX_TRACE
1895	buffer = global_trace.max_buffer.buffer;
1896	if (buffer)
1897		ring_buffer_record_disable(buffer);
1898#endif
1899
1900	arch_spin_unlock(&global_trace.max_lock);
1901
1902 out:
1903	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1904}
1905
1906static void tracing_stop_tr(struct trace_array *tr)
1907{
1908	struct ring_buffer *buffer;
1909	unsigned long flags;
1910
1911	/* If global, we need to also stop the max tracer */
1912	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1913		return tracing_stop();
1914
1915	raw_spin_lock_irqsave(&tr->start_lock, flags);
1916	if (tr->stop_count++)
1917		goto out;
1918
1919	buffer = tr->trace_buffer.buffer;
1920	if (buffer)
1921		ring_buffer_record_disable(buffer);
1922
1923 out:
1924	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1925}
1926
 
 
1927static int trace_save_cmdline(struct task_struct *tsk)
1928{
1929	unsigned pid, idx;
1930
1931	/* treat recording of idle task as a success */
1932	if (!tsk->pid)
1933		return 1;
1934
1935	if (unlikely(tsk->pid > PID_MAX_DEFAULT))
1936		return 0;
1937
1938	/*
1939	 * It's not the end of the world if we don't get
1940	 * the lock, but we also don't want to spin
1941	 * nor do we want to disable interrupts,
1942	 * so if we miss here, then better luck next time.
1943	 */
1944	if (!arch_spin_trylock(&trace_cmdline_lock))
1945		return 0;
1946
1947	idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1948	if (idx == NO_CMDLINE_MAP) {
1949		idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1950
1951		/*
1952		 * Check whether the cmdline buffer at idx has a pid
1953		 * mapped. We are going to overwrite that entry so we
1954		 * need to clear the map_pid_to_cmdline. Otherwise we
1955		 * would read the new comm for the old pid.
1956		 */
1957		pid = savedcmd->map_cmdline_to_pid[idx];
1958		if (pid != NO_CMDLINE_MAP)
1959			savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1960
1961		savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1962		savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1963
1964		savedcmd->cmdline_idx = idx;
1965	}
1966
1967	set_cmdline(idx, tsk->comm);
1968
1969	arch_spin_unlock(&trace_cmdline_lock);
1970
1971	return 1;
1972}
1973
1974static void __trace_find_cmdline(int pid, char comm[])
1975{
1976	unsigned map;
1977
1978	if (!pid) {
1979		strcpy(comm, "<idle>");
1980		return;
1981	}
1982
1983	if (WARN_ON_ONCE(pid < 0)) {
1984		strcpy(comm, "<XXX>");
1985		return;
1986	}
1987
1988	if (pid > PID_MAX_DEFAULT) {
1989		strcpy(comm, "<...>");
1990		return;
1991	}
1992
1993	map = savedcmd->map_pid_to_cmdline[pid];
1994	if (map != NO_CMDLINE_MAP)
1995		strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
1996	else
1997		strcpy(comm, "<...>");
1998}
1999
2000void trace_find_cmdline(int pid, char comm[])
2001{
2002	preempt_disable();
2003	arch_spin_lock(&trace_cmdline_lock);
2004
2005	__trace_find_cmdline(pid, comm);
2006
2007	arch_spin_unlock(&trace_cmdline_lock);
2008	preempt_enable();
2009}
2010
2011int trace_find_tgid(int pid)
2012{
2013	if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2014		return 0;
2015
2016	return tgid_map[pid];
2017}
2018
2019static int trace_save_tgid(struct task_struct *tsk)
2020{
2021	/* treat recording of idle task as a success */
2022	if (!tsk->pid)
2023		return 1;
2024
2025	if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
2026		return 0;
2027
2028	tgid_map[tsk->pid] = tsk->tgid;
2029	return 1;
2030}
2031
2032static bool tracing_record_taskinfo_skip(int flags)
2033{
2034	if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2035		return true;
2036	if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2037		return true;
2038	if (!__this_cpu_read(trace_taskinfo_save))
2039		return true;
2040	return false;
2041}
2042
2043/**
2044 * tracing_record_taskinfo - record the task info of a task
2045 *
2046 * @task  - task to record
2047 * @flags - TRACE_RECORD_CMDLINE for recording comm
2048 *        - TRACE_RECORD_TGID for recording tgid
2049 */
2050void tracing_record_taskinfo(struct task_struct *task, int flags)
2051{
2052	bool done;
2053
2054	if (tracing_record_taskinfo_skip(flags))
2055		return;
2056
2057	/*
2058	 * Record as much task information as possible. If some fail, continue
2059	 * to try to record the others.
2060	 */
2061	done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2062	done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2063
2064	/* If recording any information failed, retry again soon. */
2065	if (!done)
2066		return;
2067
2068	__this_cpu_write(trace_taskinfo_save, false);
 
2069}
2070
2071/**
2072 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2073 *
2074 * @prev - previous task during sched_switch
2075 * @next - next task during sched_switch
2076 * @flags - TRACE_RECORD_CMDLINE for recording comm
2077 *          TRACE_RECORD_TGID for recording tgid
2078 */
2079void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2080					  struct task_struct *next, int flags)
2081{
2082	bool done;
2083
2084	if (tracing_record_taskinfo_skip(flags))
2085		return;
2086
2087	/*
2088	 * Record as much task information as possible. If some fail, continue
2089	 * to try to record the others.
2090	 */
2091	done  = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2092	done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2093	done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2094	done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2095
2096	/* If recording any information failed, retry again soon. */
2097	if (!done)
2098		return;
2099
2100	__this_cpu_write(trace_taskinfo_save, false);
2101}
2102
2103/* Helpers to record a specific task information */
2104void tracing_record_cmdline(struct task_struct *task)
2105{
2106	tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2107}
2108
2109void tracing_record_tgid(struct task_struct *task)
2110{
2111	tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2112}
2113
2114/*
2115 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2116 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2117 * simplifies those functions and keeps them in sync.
2118 */
2119enum print_line_t trace_handle_return(struct trace_seq *s)
2120{
2121	return trace_seq_has_overflowed(s) ?
2122		TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2123}
2124EXPORT_SYMBOL_GPL(trace_handle_return);
2125
2126void
2127tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
2128			     int pc)
2129{
2130	struct task_struct *tsk = current;
2131
2132	entry->preempt_count		= pc & 0xff;
2133	entry->pid			= (tsk) ? tsk->pid : 0;
2134	entry->flags =
2135#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2136		(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
2137#else
2138		TRACE_FLAG_IRQS_NOSUPPORT |
2139#endif
2140		((pc & NMI_MASK    ) ? TRACE_FLAG_NMI     : 0) |
2141		((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
2142		((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
2143		(tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2144		(test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2145}
2146EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
2147
2148struct ring_buffer_event *
2149trace_buffer_lock_reserve(struct ring_buffer *buffer,
2150			  int type,
2151			  unsigned long len,
2152			  unsigned long flags, int pc)
2153{
2154	return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
2155}
2156
2157DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2158DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2159static int trace_buffered_event_ref;
2160
2161/**
2162 * trace_buffered_event_enable - enable buffering events
2163 *
2164 * When events are being filtered, it is quicker to use a temporary
2165 * buffer to write the event data into if there's a likely chance
2166 * that it will not be committed. The discard of the ring buffer
2167 * is not as fast as committing, and is much slower than copying
2168 * a commit.
2169 *
2170 * When an event is to be filtered, allocate per cpu buffers to
2171 * write the event data into, and if the event is filtered and discarded
2172 * it is simply dropped, otherwise, the entire data is to be committed
2173 * in one shot.
2174 */
2175void trace_buffered_event_enable(void)
2176{
2177	struct ring_buffer_event *event;
2178	struct page *page;
2179	int cpu;
2180
2181	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2182
2183	if (trace_buffered_event_ref++)
2184		return;
2185
2186	for_each_tracing_cpu(cpu) {
2187		page = alloc_pages_node(cpu_to_node(cpu),
2188					GFP_KERNEL | __GFP_NORETRY, 0);
2189		if (!page)
2190			goto failed;
2191
2192		event = page_address(page);
2193		memset(event, 0, sizeof(*event));
2194
2195		per_cpu(trace_buffered_event, cpu) = event;
2196
2197		preempt_disable();
2198		if (cpu == smp_processor_id() &&
2199		    this_cpu_read(trace_buffered_event) !=
2200		    per_cpu(trace_buffered_event, cpu))
2201			WARN_ON_ONCE(1);
2202		preempt_enable();
2203	}
2204
2205	return;
2206 failed:
2207	trace_buffered_event_disable();
2208}
2209
2210static void enable_trace_buffered_event(void *data)
2211{
2212	/* Probably not needed, but do it anyway */
2213	smp_rmb();
2214	this_cpu_dec(trace_buffered_event_cnt);
2215}
2216
2217static void disable_trace_buffered_event(void *data)
 
2218{
2219	this_cpu_inc(trace_buffered_event_cnt);
 
2220}
2221
2222/**
2223 * trace_buffered_event_disable - disable buffering events
2224 *
2225 * When a filter is removed, it is faster to not use the buffered
2226 * events, and to commit directly into the ring buffer. Free up
2227 * the temp buffers when there are no more users. This requires
2228 * special synchronization with current events.
2229 */
2230void trace_buffered_event_disable(void)
2231{
2232	int cpu;
2233
2234	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2235
2236	if (WARN_ON_ONCE(!trace_buffered_event_ref))
2237		return;
2238
2239	if (--trace_buffered_event_ref)
2240		return;
2241
2242	preempt_disable();
2243	/* For each CPU, set the buffer as used. */
2244	smp_call_function_many(tracing_buffer_mask,
2245			       disable_trace_buffered_event, NULL, 1);
2246	preempt_enable();
2247
2248	/* Wait for all current users to finish */
2249	synchronize_sched();
2250
2251	for_each_tracing_cpu(cpu) {
2252		free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2253		per_cpu(trace_buffered_event, cpu) = NULL;
2254	}
2255	/*
2256	 * Make sure trace_buffered_event is NULL before clearing
2257	 * trace_buffered_event_cnt.
2258	 */
2259	smp_wmb();
2260
2261	preempt_disable();
2262	/* Do the work on each cpu */
2263	smp_call_function_many(tracing_buffer_mask,
2264			       enable_trace_buffered_event, NULL, 1);
2265	preempt_enable();
2266}
 
2267
2268static struct ring_buffer *temp_buffer;
2269
2270struct ring_buffer_event *
2271trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
2272			  struct trace_event_file *trace_file,
2273			  int type, unsigned long len,
2274			  unsigned long flags, int pc)
2275{
2276	struct ring_buffer_event *entry;
2277	int val;
2278
2279	*current_rb = trace_file->tr->trace_buffer.buffer;
2280
2281	if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2282	     (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2283	    (entry = this_cpu_read(trace_buffered_event))) {
2284		/* Try to use the per cpu buffer first */
2285		val = this_cpu_inc_return(trace_buffered_event_cnt);
2286		if (val == 1) {
2287			trace_event_setup(entry, type, flags, pc);
2288			entry->array[0] = len;
2289			return entry;
2290		}
2291		this_cpu_dec(trace_buffered_event_cnt);
2292	}
2293
2294	entry = __trace_buffer_lock_reserve(*current_rb,
2295					    type, len, flags, pc);
2296	/*
2297	 * If tracing is off, but we have triggers enabled
2298	 * we still need to look at the event data. Use the temp_buffer
2299	 * to store the trace event for the tigger to use. It's recusive
2300	 * safe and will not be recorded anywhere.
2301	 */
2302	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2303		*current_rb = temp_buffer;
2304		entry = __trace_buffer_lock_reserve(*current_rb,
2305						    type, len, flags, pc);
2306	}
2307	return entry;
2308}
2309EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2310
2311static DEFINE_SPINLOCK(tracepoint_iter_lock);
2312static DEFINE_MUTEX(tracepoint_printk_mutex);
2313
2314static void output_printk(struct trace_event_buffer *fbuffer)
2315{
2316	struct trace_event_call *event_call;
2317	struct trace_event *event;
2318	unsigned long flags;
2319	struct trace_iterator *iter = tracepoint_print_iter;
2320
2321	/* We should never get here if iter is NULL */
2322	if (WARN_ON_ONCE(!iter))
2323		return;
2324
2325	event_call = fbuffer->trace_file->event_call;
2326	if (!event_call || !event_call->event.funcs ||
2327	    !event_call->event.funcs->trace)
2328		return;
2329
2330	event = &fbuffer->trace_file->event_call->event;
2331
2332	spin_lock_irqsave(&tracepoint_iter_lock, flags);
2333	trace_seq_init(&iter->seq);
2334	iter->ent = fbuffer->entry;
2335	event_call->event.funcs->trace(iter, 0, event);
2336	trace_seq_putc(&iter->seq, 0);
2337	printk("%s", iter->seq.buffer);
2338
2339	spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2340}
2341
2342int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2343			     void __user *buffer, size_t *lenp,
2344			     loff_t *ppos)
2345{
2346	int save_tracepoint_printk;
2347	int ret;
2348
2349	mutex_lock(&tracepoint_printk_mutex);
2350	save_tracepoint_printk = tracepoint_printk;
2351
2352	ret = proc_dointvec(table, write, buffer, lenp, ppos);
2353
2354	/*
2355	 * This will force exiting early, as tracepoint_printk
2356	 * is always zero when tracepoint_printk_iter is not allocated
2357	 */
2358	if (!tracepoint_print_iter)
2359		tracepoint_printk = 0;
2360
2361	if (save_tracepoint_printk == tracepoint_printk)
2362		goto out;
2363
2364	if (tracepoint_printk)
2365		static_key_enable(&tracepoint_printk_key.key);
2366	else
2367		static_key_disable(&tracepoint_printk_key.key);
2368
2369 out:
2370	mutex_unlock(&tracepoint_printk_mutex);
2371
2372	return ret;
2373}
2374
2375void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2376{
2377	if (static_key_false(&tracepoint_printk_key.key))
2378		output_printk(fbuffer);
2379
2380	event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
2381				    fbuffer->event, fbuffer->entry,
2382				    fbuffer->flags, fbuffer->pc);
2383}
2384EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2385
2386/*
2387 * Skip 3:
2388 *
2389 *   trace_buffer_unlock_commit_regs()
2390 *   trace_event_buffer_commit()
2391 *   trace_event_raw_event_xxx()
2392 */
2393# define STACK_SKIP 3
2394
2395void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2396				     struct ring_buffer *buffer,
2397				     struct ring_buffer_event *event,
2398				     unsigned long flags, int pc,
2399				     struct pt_regs *regs)
2400{
2401	__buffer_unlock_commit(buffer, event);
2402
2403	/*
2404	 * If regs is not set, then skip the necessary functions.
2405	 * Note, we can still get here via blktrace, wakeup tracer
2406	 * and mmiotrace, but that's ok if they lose a function or
2407	 * two. They are not that meaningful.
2408	 */
2409	ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
2410	ftrace_trace_userstack(buffer, flags, pc);
2411}
 
2412
2413/*
2414 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2415 */
2416void
2417trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
2418				   struct ring_buffer_event *event)
2419{
2420	__buffer_unlock_commit(buffer, event);
2421}
2422
2423static void
2424trace_process_export(struct trace_export *export,
2425	       struct ring_buffer_event *event)
2426{
2427	struct trace_entry *entry;
2428	unsigned int size = 0;
2429
2430	entry = ring_buffer_event_data(event);
2431	size = ring_buffer_event_length(event);
2432	export->write(export, entry, size);
2433}
2434
2435static DEFINE_MUTEX(ftrace_export_lock);
2436
2437static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2438
2439static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2440
2441static inline void ftrace_exports_enable(void)
2442{
2443	static_branch_enable(&ftrace_exports_enabled);
2444}
2445
2446static inline void ftrace_exports_disable(void)
2447{
2448	static_branch_disable(&ftrace_exports_enabled);
2449}
2450
2451void ftrace_exports(struct ring_buffer_event *event)
2452{
2453	struct trace_export *export;
2454
2455	preempt_disable_notrace();
2456
2457	export = rcu_dereference_raw_notrace(ftrace_exports_list);
2458	while (export) {
2459		trace_process_export(export, event);
2460		export = rcu_dereference_raw_notrace(export->next);
2461	}
2462
2463	preempt_enable_notrace();
2464}
2465
2466static inline void
2467add_trace_export(struct trace_export **list, struct trace_export *export)
2468{
2469	rcu_assign_pointer(export->next, *list);
2470	/*
2471	 * We are entering export into the list but another
2472	 * CPU might be walking that list. We need to make sure
2473	 * the export->next pointer is valid before another CPU sees
2474	 * the export pointer included into the list.
2475	 */
2476	rcu_assign_pointer(*list, export);
2477}
2478
2479static inline int
2480rm_trace_export(struct trace_export **list, struct trace_export *export)
2481{
2482	struct trace_export **p;
2483
2484	for (p = list; *p != NULL; p = &(*p)->next)
2485		if (*p == export)
2486			break;
2487
2488	if (*p != export)
2489		return -1;
2490
2491	rcu_assign_pointer(*p, (*p)->next);
2492
2493	return 0;
2494}
2495
2496static inline void
2497add_ftrace_export(struct trace_export **list, struct trace_export *export)
2498{
2499	if (*list == NULL)
2500		ftrace_exports_enable();
2501
2502	add_trace_export(list, export);
2503}
2504
2505static inline int
2506rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2507{
2508	int ret;
2509
2510	ret = rm_trace_export(list, export);
2511	if (*list == NULL)
2512		ftrace_exports_disable();
2513
2514	return ret;
2515}
2516
2517int register_ftrace_export(struct trace_export *export)
2518{
2519	if (WARN_ON_ONCE(!export->write))
2520		return -1;
2521
2522	mutex_lock(&ftrace_export_lock);
2523
2524	add_ftrace_export(&ftrace_exports_list, export);
2525
2526	mutex_unlock(&ftrace_export_lock);
2527
2528	return 0;
2529}
2530EXPORT_SYMBOL_GPL(register_ftrace_export);
2531
2532int unregister_ftrace_export(struct trace_export *export)
2533{
2534	int ret;
2535
2536	mutex_lock(&ftrace_export_lock);
2537
2538	ret = rm_ftrace_export(&ftrace_exports_list, export);
2539
2540	mutex_unlock(&ftrace_export_lock);
2541
2542	return ret;
2543}
2544EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2545
2546void
2547trace_function(struct trace_array *tr,
2548	       unsigned long ip, unsigned long parent_ip, unsigned long flags,
2549	       int pc)
2550{
2551	struct trace_event_call *call = &event_function;
2552	struct ring_buffer *buffer = tr->trace_buffer.buffer;
2553	struct ring_buffer_event *event;
2554	struct ftrace_entry *entry;
2555
2556	event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2557					    flags, pc);
2558	if (!event)
2559		return;
2560	entry	= ring_buffer_event_data(event);
2561	entry->ip			= ip;
2562	entry->parent_ip		= parent_ip;
2563
2564	if (!call_filter_check_discard(call, entry, buffer, event)) {
2565		if (static_branch_unlikely(&ftrace_exports_enabled))
2566			ftrace_exports(event);
2567		__buffer_unlock_commit(buffer, event);
2568	}
2569}
2570
2571#ifdef CONFIG_STACKTRACE
2572
2573#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2574struct ftrace_stack {
2575	unsigned long		calls[FTRACE_STACK_MAX_ENTRIES];
2576};
2577
2578static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
2579static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2580
2581static void __ftrace_trace_stack(struct ring_buffer *buffer,
2582				 unsigned long flags,
2583				 int skip, int pc, struct pt_regs *regs)
2584{
2585	struct trace_event_call *call = &event_kernel_stack;
2586	struct ring_buffer_event *event;
2587	struct stack_entry *entry;
2588	struct stack_trace trace;
2589	int use_stack;
2590	int size = FTRACE_STACK_ENTRIES;
2591
2592	trace.nr_entries	= 0;
2593	trace.skip		= skip;
2594
2595	/*
2596	 * Add one, for this function and the call to save_stack_trace()
2597	 * If regs is set, then these functions will not be in the way.
2598	 */
2599#ifndef CONFIG_UNWINDER_ORC
2600	if (!regs)
2601		trace.skip++;
2602#endif
2603
2604	/*
2605	 * Since events can happen in NMIs there's no safe way to
2606	 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2607	 * or NMI comes in, it will just have to use the default
2608	 * FTRACE_STACK_SIZE.
2609	 */
2610	preempt_disable_notrace();
2611
2612	use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
2613	/*
2614	 * We don't need any atomic variables, just a barrier.
2615	 * If an interrupt comes in, we don't care, because it would
2616	 * have exited and put the counter back to what we want.
2617	 * We just need a barrier to keep gcc from moving things
2618	 * around.
2619	 */
2620	barrier();
2621	if (use_stack == 1) {
2622		trace.entries		= this_cpu_ptr(ftrace_stack.calls);
2623		trace.max_entries	= FTRACE_STACK_MAX_ENTRIES;
2624
2625		if (regs)
2626			save_stack_trace_regs(regs, &trace);
2627		else
2628			save_stack_trace(&trace);
2629
2630		if (trace.nr_entries > size)
2631			size = trace.nr_entries;
2632	} else
2633		/* From now on, use_stack is a boolean */
2634		use_stack = 0;
2635
2636	size *= sizeof(unsigned long);
2637
2638	event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2639					    sizeof(*entry) + size, flags, pc);
2640	if (!event)
2641		goto out;
2642	entry = ring_buffer_event_data(event);
2643
2644	memset(&entry->caller, 0, size);
2645
2646	if (use_stack)
2647		memcpy(&entry->caller, trace.entries,
2648		       trace.nr_entries * sizeof(unsigned long));
2649	else {
2650		trace.max_entries	= FTRACE_STACK_ENTRIES;
2651		trace.entries		= entry->caller;
2652		if (regs)
2653			save_stack_trace_regs(regs, &trace);
2654		else
2655			save_stack_trace(&trace);
2656	}
2657
2658	entry->size = trace.nr_entries;
2659
2660	if (!call_filter_check_discard(call, entry, buffer, event))
2661		__buffer_unlock_commit(buffer, event);
2662
2663 out:
2664	/* Again, don't let gcc optimize things here */
2665	barrier();
2666	__this_cpu_dec(ftrace_stack_reserve);
2667	preempt_enable_notrace();
2668
2669}
2670
2671static inline void ftrace_trace_stack(struct trace_array *tr,
2672				      struct ring_buffer *buffer,
2673				      unsigned long flags,
2674				      int skip, int pc, struct pt_regs *regs)
2675{
2676	if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2677		return;
2678
2679	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
2680}
2681
2682void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2683		   int pc)
2684{
2685	struct ring_buffer *buffer = tr->trace_buffer.buffer;
2686
2687	if (rcu_is_watching()) {
2688		__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2689		return;
2690	}
2691
2692	/*
2693	 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
2694	 * but if the above rcu_is_watching() failed, then the NMI
2695	 * triggered someplace critical, and rcu_irq_enter() should
2696	 * not be called from NMI.
2697	 */
2698	if (unlikely(in_nmi()))
2699		return;
2700
2701	rcu_irq_enter_irqson();
2702	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2703	rcu_irq_exit_irqson();
2704}
2705
2706/**
2707 * trace_dump_stack - record a stack back trace in the trace buffer
2708 * @skip: Number of functions to skip (helper handlers)
2709 */
2710void trace_dump_stack(int skip)
2711{
2712	unsigned long flags;
2713
2714	if (tracing_disabled || tracing_selftest_running)
2715		return;
2716
2717	local_save_flags(flags);
2718
2719#ifndef CONFIG_UNWINDER_ORC
2720	/* Skip 1 to skip this function. */
2721	skip++;
2722#endif
 
2723	__ftrace_trace_stack(global_trace.trace_buffer.buffer,
2724			     flags, skip, preempt_count(), NULL);
2725}
2726
2727static DEFINE_PER_CPU(int, user_stack_count);
2728
2729void
2730ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
2731{
2732	struct trace_event_call *call = &event_user_stack;
2733	struct ring_buffer_event *event;
2734	struct userstack_entry *entry;
2735	struct stack_trace trace;
2736
2737	if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
2738		return;
2739
2740	/*
2741	 * NMIs can not handle page faults, even with fix ups.
2742	 * The save user stack can (and often does) fault.
2743	 */
2744	if (unlikely(in_nmi()))
2745		return;
2746
2747	/*
2748	 * prevent recursion, since the user stack tracing may
2749	 * trigger other kernel events.
2750	 */
2751	preempt_disable();
2752	if (__this_cpu_read(user_stack_count))
2753		goto out;
2754
2755	__this_cpu_inc(user_stack_count);
2756
2757	event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2758					    sizeof(*entry), flags, pc);
2759	if (!event)
2760		goto out_drop_count;
2761	entry	= ring_buffer_event_data(event);
2762
2763	entry->tgid		= current->tgid;
2764	memset(&entry->caller, 0, sizeof(entry->caller));
2765
2766	trace.nr_entries	= 0;
2767	trace.max_entries	= FTRACE_STACK_ENTRIES;
2768	trace.skip		= 0;
2769	trace.entries		= entry->caller;
2770
2771	save_stack_trace_user(&trace);
2772	if (!call_filter_check_discard(call, entry, buffer, event))
2773		__buffer_unlock_commit(buffer, event);
2774
2775 out_drop_count:
2776	__this_cpu_dec(user_stack_count);
2777 out:
2778	preempt_enable();
2779}
2780
2781#ifdef UNUSED
2782static void __trace_userstack(struct trace_array *tr, unsigned long flags)
2783{
2784	ftrace_trace_userstack(tr, flags, preempt_count());
2785}
2786#endif /* UNUSED */
2787
2788#endif /* CONFIG_STACKTRACE */
2789
2790/* created for use with alloc_percpu */
2791struct trace_buffer_struct {
2792	int nesting;
2793	char buffer[4][TRACE_BUF_SIZE];
2794};
2795
2796static struct trace_buffer_struct *trace_percpu_buffer;
 
 
 
2797
2798/*
2799 * Thise allows for lockless recording.  If we're nested too deeply, then
2800 * this returns NULL.
 
 
 
2801 */
2802static char *get_trace_buf(void)
2803{
2804	struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
2805
2806	if (!buffer || buffer->nesting >= 4)
2807		return NULL;
 
 
 
 
 
 
 
 
 
 
2808
2809	buffer->nesting++;
 
2810
2811	/* Interrupts must see nesting incremented before we use the buffer */
2812	barrier();
2813	return &buffer->buffer[buffer->nesting][0];
2814}
2815
2816static void put_trace_buf(void)
2817{
2818	/* Don't let the decrement of nesting leak before this */
2819	barrier();
2820	this_cpu_dec(trace_percpu_buffer->nesting);
2821}
2822
2823static int alloc_percpu_trace_buffer(void)
2824{
2825	struct trace_buffer_struct *buffers;
 
 
 
2826
2827	buffers = alloc_percpu(struct trace_buffer_struct);
2828	if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
2829		return -ENOMEM;
 
 
 
 
 
 
 
 
 
 
 
 
2830
2831	trace_percpu_buffer = buffers;
 
 
 
 
2832	return 0;
 
 
 
 
 
 
 
 
 
 
2833}
2834
2835static int buffers_allocated;
2836
2837void trace_printk_init_buffers(void)
2838{
2839	if (buffers_allocated)
2840		return;
2841
2842	if (alloc_percpu_trace_buffer())
2843		return;
2844
2845	/* trace_printk() is for debug use only. Don't use it in production. */
2846
2847	pr_warn("\n");
2848	pr_warn("**********************************************************\n");
2849	pr_warn("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
2850	pr_warn("**                                                      **\n");
2851	pr_warn("** trace_printk() being used. Allocating extra memory.  **\n");
2852	pr_warn("**                                                      **\n");
2853	pr_warn("** This means that this is a DEBUG kernel and it is     **\n");
2854	pr_warn("** unsafe for production use.                           **\n");
2855	pr_warn("**                                                      **\n");
2856	pr_warn("** If you see this message and you are not debugging    **\n");
2857	pr_warn("** the kernel, report this immediately to your vendor!  **\n");
2858	pr_warn("**                                                      **\n");
2859	pr_warn("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
2860	pr_warn("**********************************************************\n");
2861
2862	/* Expand the buffers to set size */
2863	tracing_update_buffers();
2864
2865	buffers_allocated = 1;
2866
2867	/*
2868	 * trace_printk_init_buffers() can be called by modules.
2869	 * If that happens, then we need to start cmdline recording
2870	 * directly here. If the global_trace.buffer is already
2871	 * allocated here, then this was called by module code.
2872	 */
2873	if (global_trace.trace_buffer.buffer)
2874		tracing_start_cmdline_record();
2875}
2876
2877void trace_printk_start_comm(void)
2878{
2879	/* Start tracing comms if trace printk is set */
2880	if (!buffers_allocated)
2881		return;
2882	tracing_start_cmdline_record();
2883}
2884
2885static void trace_printk_start_stop_comm(int enabled)
2886{
2887	if (!buffers_allocated)
2888		return;
2889
2890	if (enabled)
2891		tracing_start_cmdline_record();
2892	else
2893		tracing_stop_cmdline_record();
2894}
2895
2896/**
2897 * trace_vbprintk - write binary msg to tracing buffer
2898 *
2899 */
2900int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2901{
2902	struct trace_event_call *call = &event_bprint;
2903	struct ring_buffer_event *event;
2904	struct ring_buffer *buffer;
2905	struct trace_array *tr = &global_trace;
2906	struct bprint_entry *entry;
2907	unsigned long flags;
2908	char *tbuffer;
2909	int len = 0, size, pc;
2910
2911	if (unlikely(tracing_selftest_running || tracing_disabled))
2912		return 0;
2913
2914	/* Don't pollute graph traces with trace_vprintk internals */
2915	pause_graph_tracing();
2916
2917	pc = preempt_count();
2918	preempt_disable_notrace();
2919
2920	tbuffer = get_trace_buf();
2921	if (!tbuffer) {
2922		len = 0;
2923		goto out_nobuffer;
2924	}
2925
2926	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2927
2928	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2929		goto out;
2930
2931	local_save_flags(flags);
2932	size = sizeof(*entry) + sizeof(u32) * len;
2933	buffer = tr->trace_buffer.buffer;
2934	event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2935					    flags, pc);
2936	if (!event)
2937		goto out;
2938	entry = ring_buffer_event_data(event);
2939	entry->ip			= ip;
2940	entry->fmt			= fmt;
2941
2942	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2943	if (!call_filter_check_discard(call, entry, buffer, event)) {
2944		__buffer_unlock_commit(buffer, event);
2945		ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2946	}
2947
2948out:
2949	put_trace_buf();
2950
2951out_nobuffer:
2952	preempt_enable_notrace();
2953	unpause_graph_tracing();
2954
2955	return len;
2956}
2957EXPORT_SYMBOL_GPL(trace_vbprintk);
2958
2959static int
2960__trace_array_vprintk(struct ring_buffer *buffer,
2961		      unsigned long ip, const char *fmt, va_list args)
2962{
2963	struct trace_event_call *call = &event_print;
2964	struct ring_buffer_event *event;
2965	int len = 0, size, pc;
2966	struct print_entry *entry;
2967	unsigned long flags;
2968	char *tbuffer;
2969
2970	if (tracing_disabled || tracing_selftest_running)
2971		return 0;
2972
2973	/* Don't pollute graph traces with trace_vprintk internals */
2974	pause_graph_tracing();
2975
2976	pc = preempt_count();
2977	preempt_disable_notrace();
2978
2979
2980	tbuffer = get_trace_buf();
2981	if (!tbuffer) {
2982		len = 0;
2983		goto out_nobuffer;
2984	}
2985
2986	len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2987
2988	local_save_flags(flags);
2989	size = sizeof(*entry) + len + 1;
2990	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2991					    flags, pc);
2992	if (!event)
2993		goto out;
2994	entry = ring_buffer_event_data(event);
2995	entry->ip = ip;
2996
2997	memcpy(&entry->buf, tbuffer, len + 1);
2998	if (!call_filter_check_discard(call, entry, buffer, event)) {
2999		__buffer_unlock_commit(buffer, event);
3000		ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
3001	}
3002
3003out:
3004	put_trace_buf();
3005
3006out_nobuffer:
3007	preempt_enable_notrace();
3008	unpause_graph_tracing();
3009
3010	return len;
3011}
3012
3013int trace_array_vprintk(struct trace_array *tr,
3014			unsigned long ip, const char *fmt, va_list args)
3015{
3016	return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
3017}
3018
3019int trace_array_printk(struct trace_array *tr,
3020		       unsigned long ip, const char *fmt, ...)
3021{
3022	int ret;
3023	va_list ap;
3024
3025	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3026		return 0;
3027
3028	va_start(ap, fmt);
3029	ret = trace_array_vprintk(tr, ip, fmt, ap);
3030	va_end(ap);
3031	return ret;
3032}
3033
3034int trace_array_printk_buf(struct ring_buffer *buffer,
3035			   unsigned long ip, const char *fmt, ...)
3036{
3037	int ret;
3038	va_list ap;
3039
3040	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3041		return 0;
3042
3043	va_start(ap, fmt);
3044	ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3045	va_end(ap);
3046	return ret;
3047}
3048
3049int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3050{
3051	return trace_array_vprintk(&global_trace, ip, fmt, args);
3052}
3053EXPORT_SYMBOL_GPL(trace_vprintk);
3054
3055static void trace_iterator_increment(struct trace_iterator *iter)
3056{
3057	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3058
3059	iter->idx++;
3060	if (buf_iter)
3061		ring_buffer_read(buf_iter, NULL);
3062}
3063
3064static struct trace_entry *
3065peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3066		unsigned long *lost_events)
3067{
3068	struct ring_buffer_event *event;
3069	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3070
3071	if (buf_iter)
3072		event = ring_buffer_iter_peek(buf_iter, ts);
3073	else
3074		event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
3075					 lost_events);
3076
3077	if (event) {
3078		iter->ent_size = ring_buffer_event_length(event);
3079		return ring_buffer_event_data(event);
3080	}
3081	iter->ent_size = 0;
3082	return NULL;
3083}
3084
3085static struct trace_entry *
3086__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3087		  unsigned long *missing_events, u64 *ent_ts)
3088{
3089	struct ring_buffer *buffer = iter->trace_buffer->buffer;
3090	struct trace_entry *ent, *next = NULL;
3091	unsigned long lost_events = 0, next_lost = 0;
3092	int cpu_file = iter->cpu_file;
3093	u64 next_ts = 0, ts;
3094	int next_cpu = -1;
3095	int next_size = 0;
3096	int cpu;
3097
3098	/*
3099	 * If we are in a per_cpu trace file, don't bother by iterating over
3100	 * all cpu and peek directly.
3101	 */
3102	if (cpu_file > RING_BUFFER_ALL_CPUS) {
3103		if (ring_buffer_empty_cpu(buffer, cpu_file))
3104			return NULL;
3105		ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3106		if (ent_cpu)
3107			*ent_cpu = cpu_file;
3108
3109		return ent;
3110	}
3111
3112	for_each_tracing_cpu(cpu) {
3113
3114		if (ring_buffer_empty_cpu(buffer, cpu))
3115			continue;
3116
3117		ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3118
3119		/*
3120		 * Pick the entry with the smallest timestamp:
3121		 */
3122		if (ent && (!next || ts < next_ts)) {
3123			next = ent;
3124			next_cpu = cpu;
3125			next_ts = ts;
3126			next_lost = lost_events;
3127			next_size = iter->ent_size;
3128		}
3129	}
3130
3131	iter->ent_size = next_size;
3132
3133	if (ent_cpu)
3134		*ent_cpu = next_cpu;
3135
3136	if (ent_ts)
3137		*ent_ts = next_ts;
3138
3139	if (missing_events)
3140		*missing_events = next_lost;
3141
3142	return next;
3143}
3144
3145/* Find the next real entry, without updating the iterator itself */
3146struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3147					  int *ent_cpu, u64 *ent_ts)
3148{
3149	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3150}
3151
3152/* Find the next real entry, and increment the iterator to the next entry */
3153void *trace_find_next_entry_inc(struct trace_iterator *iter)
3154{
3155	iter->ent = __find_next_entry(iter, &iter->cpu,
3156				      &iter->lost_events, &iter->ts);
3157
3158	if (iter->ent)
3159		trace_iterator_increment(iter);
3160
3161	return iter->ent ? iter : NULL;
3162}
3163
3164static void trace_consume(struct trace_iterator *iter)
3165{
3166	ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
3167			    &iter->lost_events);
3168}
3169
3170static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3171{
3172	struct trace_iterator *iter = m->private;
3173	int i = (int)*pos;
3174	void *ent;
3175
3176	WARN_ON_ONCE(iter->leftover);
3177
3178	(*pos)++;
3179
3180	/* can't go backwards */
3181	if (iter->idx > i)
3182		return NULL;
3183
3184	if (iter->idx < 0)
3185		ent = trace_find_next_entry_inc(iter);
3186	else
3187		ent = iter;
3188
3189	while (ent && iter->idx < i)
3190		ent = trace_find_next_entry_inc(iter);
3191
3192	iter->pos = *pos;
3193
3194	return ent;
3195}
3196
3197void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3198{
3199	struct ring_buffer_event *event;
3200	struct ring_buffer_iter *buf_iter;
3201	unsigned long entries = 0;
3202	u64 ts;
3203
3204	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
3205
3206	buf_iter = trace_buffer_iter(iter, cpu);
3207	if (!buf_iter)
3208		return;
3209
3210	ring_buffer_iter_reset(buf_iter);
3211
3212	/*
3213	 * We could have the case with the max latency tracers
3214	 * that a reset never took place on a cpu. This is evident
3215	 * by the timestamp being before the start of the buffer.
3216	 */
3217	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
3218		if (ts >= iter->trace_buffer->time_start)
3219			break;
3220		entries++;
3221		ring_buffer_read(buf_iter, NULL);
3222	}
3223
3224	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
3225}
3226
3227/*
3228 * The current tracer is copied to avoid a global locking
3229 * all around.
3230 */
3231static void *s_start(struct seq_file *m, loff_t *pos)
3232{
3233	struct trace_iterator *iter = m->private;
3234	struct trace_array *tr = iter->tr;
3235	int cpu_file = iter->cpu_file;
3236	void *p = NULL;
3237	loff_t l = 0;
3238	int cpu;
3239
3240	/*
3241	 * copy the tracer to avoid using a global lock all around.
3242	 * iter->trace is a copy of current_trace, the pointer to the
3243	 * name may be used instead of a strcmp(), as iter->trace->name
3244	 * will point to the same string as current_trace->name.
3245	 */
3246	mutex_lock(&trace_types_lock);
3247	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3248		*iter->trace = *tr->current_trace;
3249	mutex_unlock(&trace_types_lock);
3250
3251#ifdef CONFIG_TRACER_MAX_TRACE
3252	if (iter->snapshot && iter->trace->use_max_tr)
3253		return ERR_PTR(-EBUSY);
3254#endif
3255
3256	if (!iter->snapshot)
3257		atomic_inc(&trace_record_taskinfo_disabled);
3258
3259	if (*pos != iter->pos) {
3260		iter->ent = NULL;
3261		iter->cpu = 0;
3262		iter->idx = -1;
3263
3264		if (cpu_file == RING_BUFFER_ALL_CPUS) {
3265			for_each_tracing_cpu(cpu)
3266				tracing_iter_reset(iter, cpu);
3267		} else
3268			tracing_iter_reset(iter, cpu_file);
3269
3270		iter->leftover = 0;
3271		for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3272			;
3273
3274	} else {
3275		/*
3276		 * If we overflowed the seq_file before, then we want
3277		 * to just reuse the trace_seq buffer again.
3278		 */
3279		if (iter->leftover)
3280			p = iter;
3281		else {
3282			l = *pos - 1;
3283			p = s_next(m, p, &l);
3284		}
3285	}
3286
3287	trace_event_read_lock();
3288	trace_access_lock(cpu_file);
3289	return p;
3290}
3291
3292static void s_stop(struct seq_file *m, void *p)
3293{
3294	struct trace_iterator *iter = m->private;
3295
3296#ifdef CONFIG_TRACER_MAX_TRACE
3297	if (iter->snapshot && iter->trace->use_max_tr)
3298		return;
3299#endif
3300
3301	if (!iter->snapshot)
3302		atomic_dec(&trace_record_taskinfo_disabled);
3303
3304	trace_access_unlock(iter->cpu_file);
3305	trace_event_read_unlock();
3306}
3307
3308static void
3309get_total_entries(struct trace_buffer *buf,
3310		  unsigned long *total, unsigned long *entries)
3311{
3312	unsigned long count;
3313	int cpu;
3314
3315	*total = 0;
3316	*entries = 0;
3317
3318	for_each_tracing_cpu(cpu) {
3319		count = ring_buffer_entries_cpu(buf->buffer, cpu);
3320		/*
3321		 * If this buffer has skipped entries, then we hold all
3322		 * entries for the trace and we need to ignore the
3323		 * ones before the time stamp.
3324		 */
3325		if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3326			count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3327			/* total is the same as the entries */
3328			*total += count;
3329		} else
3330			*total += count +
3331				ring_buffer_overrun_cpu(buf->buffer, cpu);
3332		*entries += count;
3333	}
3334}
3335
3336static void print_lat_help_header(struct seq_file *m)
3337{
3338	seq_puts(m, "#                  _------=> CPU#            \n"
3339		    "#                 / _-----=> irqs-off        \n"
3340		    "#                | / _----=> need-resched    \n"
3341		    "#                || / _---=> hardirq/softirq \n"
3342		    "#                ||| / _--=> preempt-depth   \n"
3343		    "#                |||| /     delay            \n"
3344		    "#  cmd     pid   ||||| time  |   caller      \n"
3345		    "#     \\   /      |||||  \\    |   /         \n");
3346}
3347
3348static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
3349{
3350	unsigned long total;
3351	unsigned long entries;
3352
3353	get_total_entries(buf, &total, &entries);
3354	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
3355		   entries, total, num_online_cpus());
3356	seq_puts(m, "#\n");
3357}
3358
3359static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m,
3360				   unsigned int flags)
3361{
3362	bool tgid = flags & TRACE_ITER_RECORD_TGID;
3363
3364	print_event_info(buf, m);
3365
3366	seq_printf(m, "#           TASK-PID   CPU#   %s  TIMESTAMP  FUNCTION\n", tgid ? "TGID     " : "");
3367	seq_printf(m, "#              | |       |    %s     |         |\n",	 tgid ? "  |      " : "");
3368}
3369
3370static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m,
3371				       unsigned int flags)
3372{
3373	bool tgid = flags & TRACE_ITER_RECORD_TGID;
3374	const char tgid_space[] = "          ";
3375	const char space[] = "  ";
3376
3377	seq_printf(m, "#                          %s  _-----=> irqs-off\n",
3378		   tgid ? tgid_space : space);
3379	seq_printf(m, "#                          %s / _----=> need-resched\n",
3380		   tgid ? tgid_space : space);
3381	seq_printf(m, "#                          %s| / _---=> hardirq/softirq\n",
3382		   tgid ? tgid_space : space);
3383	seq_printf(m, "#                          %s|| / _--=> preempt-depth\n",
3384		   tgid ? tgid_space : space);
3385	seq_printf(m, "#                          %s||| /     delay\n",
3386		   tgid ? tgid_space : space);
3387	seq_printf(m, "#           TASK-PID   CPU#%s||||    TIMESTAMP  FUNCTION\n",
3388		   tgid ? "   TGID   " : space);
3389	seq_printf(m, "#              | |       | %s||||       |         |\n",
3390		   tgid ? "     |    " : space);
3391}
3392
3393void
3394print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3395{
3396	unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3397	struct trace_buffer *buf = iter->trace_buffer;
3398	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3399	struct tracer *type = iter->trace;
3400	unsigned long entries;
3401	unsigned long total;
3402	const char *name = "preemption";
3403
3404	name = type->name;
3405
3406	get_total_entries(buf, &total, &entries);
3407
3408	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3409		   name, UTS_RELEASE);
3410	seq_puts(m, "# -----------------------------------"
3411		 "---------------------------------\n");
3412	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3413		   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3414		   nsecs_to_usecs(data->saved_latency),
3415		   entries,
3416		   total,
3417		   buf->cpu,
3418#if defined(CONFIG_PREEMPT_NONE)
3419		   "server",
3420#elif defined(CONFIG_PREEMPT_VOLUNTARY)
3421		   "desktop",
3422#elif defined(CONFIG_PREEMPT)
3423		   "preempt",
3424#else
3425		   "unknown",
3426#endif
3427		   /* These are reserved for later use */
3428		   0, 0, 0, 0);
3429#ifdef CONFIG_SMP
3430	seq_printf(m, " #P:%d)\n", num_online_cpus());
3431#else
3432	seq_puts(m, ")\n");
3433#endif
3434	seq_puts(m, "#    -----------------\n");
3435	seq_printf(m, "#    | task: %.16s-%d "
3436		   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3437		   data->comm, data->pid,
3438		   from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3439		   data->policy, data->rt_priority);
3440	seq_puts(m, "#    -----------------\n");
3441
3442	if (data->critical_start) {
3443		seq_puts(m, "#  => started at: ");
3444		seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3445		trace_print_seq(m, &iter->seq);
3446		seq_puts(m, "\n#  => ended at:   ");
3447		seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3448		trace_print_seq(m, &iter->seq);
3449		seq_puts(m, "\n#\n");
3450	}
3451
3452	seq_puts(m, "#\n");
3453}
3454
3455static void test_cpu_buff_start(struct trace_iterator *iter)
3456{
3457	struct trace_seq *s = &iter->seq;
3458	struct trace_array *tr = iter->tr;
3459
3460	if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3461		return;
3462
3463	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3464		return;
3465
3466	if (cpumask_available(iter->started) &&
3467	    cpumask_test_cpu(iter->cpu, iter->started))
3468		return;
3469
3470	if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
3471		return;
3472
3473	if (cpumask_available(iter->started))
3474		cpumask_set_cpu(iter->cpu, iter->started);
3475
3476	/* Don't print started cpu buffer for the first entry of the trace */
3477	if (iter->idx > 1)
3478		trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3479				iter->cpu);
3480}
3481
3482static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3483{
3484	struct trace_array *tr = iter->tr;
3485	struct trace_seq *s = &iter->seq;
3486	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3487	struct trace_entry *entry;
3488	struct trace_event *event;
3489
3490	entry = iter->ent;
3491
3492	test_cpu_buff_start(iter);
3493
3494	event = ftrace_find_event(entry->type);
3495
3496	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3497		if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3498			trace_print_lat_context(iter);
3499		else
3500			trace_print_context(iter);
3501	}
3502
3503	if (trace_seq_has_overflowed(s))
3504		return TRACE_TYPE_PARTIAL_LINE;
3505
3506	if (event)
3507		return event->funcs->trace(iter, sym_flags, event);
3508
3509	trace_seq_printf(s, "Unknown type %d\n", entry->type);
3510
3511	return trace_handle_return(s);
3512}
3513
3514static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3515{
3516	struct trace_array *tr = iter->tr;
3517	struct trace_seq *s = &iter->seq;
3518	struct trace_entry *entry;
3519	struct trace_event *event;
3520
3521	entry = iter->ent;
3522
3523	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3524		trace_seq_printf(s, "%d %d %llu ",
3525				 entry->pid, iter->cpu, iter->ts);
3526
3527	if (trace_seq_has_overflowed(s))
3528		return TRACE_TYPE_PARTIAL_LINE;
3529
3530	event = ftrace_find_event(entry->type);
3531	if (event)
3532		return event->funcs->raw(iter, 0, event);
3533
3534	trace_seq_printf(s, "%d ?\n", entry->type);
3535
3536	return trace_handle_return(s);
3537}
3538
3539static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3540{
3541	struct trace_array *tr = iter->tr;
3542	struct trace_seq *s = &iter->seq;
3543	unsigned char newline = '\n';
3544	struct trace_entry *entry;
3545	struct trace_event *event;
3546
3547	entry = iter->ent;
3548
3549	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3550		SEQ_PUT_HEX_FIELD(s, entry->pid);
3551		SEQ_PUT_HEX_FIELD(s, iter->cpu);
3552		SEQ_PUT_HEX_FIELD(s, iter->ts);
3553		if (trace_seq_has_overflowed(s))
3554			return TRACE_TYPE_PARTIAL_LINE;
3555	}
3556
3557	event = ftrace_find_event(entry->type);
3558	if (event) {
3559		enum print_line_t ret = event->funcs->hex(iter, 0, event);
3560		if (ret != TRACE_TYPE_HANDLED)
3561			return ret;
3562	}
3563
3564	SEQ_PUT_FIELD(s, newline);
3565
3566	return trace_handle_return(s);
3567}
3568
3569static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3570{
3571	struct trace_array *tr = iter->tr;
3572	struct trace_seq *s = &iter->seq;
3573	struct trace_entry *entry;
3574	struct trace_event *event;
3575
3576	entry = iter->ent;
3577
3578	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3579		SEQ_PUT_FIELD(s, entry->pid);
3580		SEQ_PUT_FIELD(s, iter->cpu);
3581		SEQ_PUT_FIELD(s, iter->ts);
3582		if (trace_seq_has_overflowed(s))
3583			return TRACE_TYPE_PARTIAL_LINE;
3584	}
3585
3586	event = ftrace_find_event(entry->type);
3587	return event ? event->funcs->binary(iter, 0, event) :
3588		TRACE_TYPE_HANDLED;
3589}
3590
3591int trace_empty(struct trace_iterator *iter)
3592{
3593	struct ring_buffer_iter *buf_iter;
3594	int cpu;
3595
3596	/* If we are looking at one CPU buffer, only check that one */
3597	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3598		cpu = iter->cpu_file;
3599		buf_iter = trace_buffer_iter(iter, cpu);
3600		if (buf_iter) {
3601			if (!ring_buffer_iter_empty(buf_iter))
3602				return 0;
3603		} else {
3604			if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3605				return 0;
3606		}
3607		return 1;
3608	}
3609
3610	for_each_tracing_cpu(cpu) {
3611		buf_iter = trace_buffer_iter(iter, cpu);
3612		if (buf_iter) {
3613			if (!ring_buffer_iter_empty(buf_iter))
3614				return 0;
3615		} else {
3616			if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3617				return 0;
3618		}
3619	}
3620
3621	return 1;
3622}
3623
3624/*  Called with trace_event_read_lock() held. */
3625enum print_line_t print_trace_line(struct trace_iterator *iter)
3626{
3627	struct trace_array *tr = iter->tr;
3628	unsigned long trace_flags = tr->trace_flags;
3629	enum print_line_t ret;
3630
3631	if (iter->lost_events) {
3632		trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3633				 iter->cpu, iter->lost_events);
3634		if (trace_seq_has_overflowed(&iter->seq))
3635			return TRACE_TYPE_PARTIAL_LINE;
3636	}
3637
3638	if (iter->trace && iter->trace->print_line) {
3639		ret = iter->trace->print_line(iter);
3640		if (ret != TRACE_TYPE_UNHANDLED)
3641			return ret;
3642	}
3643
3644	if (iter->ent->type == TRACE_BPUTS &&
3645			trace_flags & TRACE_ITER_PRINTK &&
3646			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3647		return trace_print_bputs_msg_only(iter);
3648
3649	if (iter->ent->type == TRACE_BPRINT &&
3650			trace_flags & TRACE_ITER_PRINTK &&
3651			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3652		return trace_print_bprintk_msg_only(iter);
3653
3654	if (iter->ent->type == TRACE_PRINT &&
3655			trace_flags & TRACE_ITER_PRINTK &&
3656			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3657		return trace_print_printk_msg_only(iter);
3658
3659	if (trace_flags & TRACE_ITER_BIN)
3660		return print_bin_fmt(iter);
3661
3662	if (trace_flags & TRACE_ITER_HEX)
3663		return print_hex_fmt(iter);
3664
3665	if (trace_flags & TRACE_ITER_RAW)
3666		return print_raw_fmt(iter);
3667
3668	return print_trace_fmt(iter);
3669}
3670
3671void trace_latency_header(struct seq_file *m)
3672{
3673	struct trace_iterator *iter = m->private;
3674	struct trace_array *tr = iter->tr;
3675
3676	/* print nothing if the buffers are empty */
3677	if (trace_empty(iter))
3678		return;
3679
3680	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3681		print_trace_header(m, iter);
3682
3683	if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
3684		print_lat_help_header(m);
3685}
3686
3687void trace_default_header(struct seq_file *m)
3688{
3689	struct trace_iterator *iter = m->private;
3690	struct trace_array *tr = iter->tr;
3691	unsigned long trace_flags = tr->trace_flags;
3692
3693	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3694		return;
3695
3696	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3697		/* print nothing if the buffers are empty */
3698		if (trace_empty(iter))
3699			return;
3700		print_trace_header(m, iter);
3701		if (!(trace_flags & TRACE_ITER_VERBOSE))
3702			print_lat_help_header(m);
3703	} else {
3704		if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3705			if (trace_flags & TRACE_ITER_IRQ_INFO)
3706				print_func_help_header_irq(iter->trace_buffer,
3707							   m, trace_flags);
3708			else
3709				print_func_help_header(iter->trace_buffer, m,
3710						       trace_flags);
3711		}
3712	}
3713}
3714
3715static void test_ftrace_alive(struct seq_file *m)
3716{
3717	if (!ftrace_is_dead())
3718		return;
3719	seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3720		    "#          MAY BE MISSING FUNCTION EVENTS\n");
3721}
3722
3723#ifdef CONFIG_TRACER_MAX_TRACE
3724static void show_snapshot_main_help(struct seq_file *m)
3725{
3726	seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3727		    "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3728		    "#                      Takes a snapshot of the main buffer.\n"
3729		    "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3730		    "#                      (Doesn't have to be '2' works with any number that\n"
3731		    "#                       is not a '0' or '1')\n");
3732}
3733
3734static void show_snapshot_percpu_help(struct seq_file *m)
3735{
3736	seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3737#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3738	seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3739		    "#                      Takes a snapshot of the main buffer for this cpu.\n");
3740#else
3741	seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3742		    "#                     Must use main snapshot file to allocate.\n");
3743#endif
3744	seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3745		    "#                      (Doesn't have to be '2' works with any number that\n"
3746		    "#                       is not a '0' or '1')\n");
3747}
3748
3749static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3750{
3751	if (iter->tr->allocated_snapshot)
3752		seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3753	else
3754		seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3755
3756	seq_puts(m, "# Snapshot commands:\n");
3757	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3758		show_snapshot_main_help(m);
3759	else
3760		show_snapshot_percpu_help(m);
3761}
3762#else
3763/* Should never be called */
3764static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3765#endif
3766
3767static int s_show(struct seq_file *m, void *v)
3768{
3769	struct trace_iterator *iter = v;
3770	int ret;
3771
3772	if (iter->ent == NULL) {
3773		if (iter->tr) {
3774			seq_printf(m, "# tracer: %s\n", iter->trace->name);
3775			seq_puts(m, "#\n");
3776			test_ftrace_alive(m);
3777		}
3778		if (iter->snapshot && trace_empty(iter))
3779			print_snapshot_help(m, iter);
3780		else if (iter->trace && iter->trace->print_header)
3781			iter->trace->print_header(m);
3782		else
3783			trace_default_header(m);
3784
3785	} else if (iter->leftover) {
3786		/*
3787		 * If we filled the seq_file buffer earlier, we
3788		 * want to just show it now.
3789		 */
3790		ret = trace_print_seq(m, &iter->seq);
3791
3792		/* ret should this time be zero, but you never know */
3793		iter->leftover = ret;
3794
3795	} else {
3796		print_trace_line(iter);
3797		ret = trace_print_seq(m, &iter->seq);
3798		/*
3799		 * If we overflow the seq_file buffer, then it will
3800		 * ask us for this data again at start up.
3801		 * Use that instead.
3802		 *  ret is 0 if seq_file write succeeded.
3803		 *        -1 otherwise.
3804		 */
3805		iter->leftover = ret;
3806	}
3807
3808	return 0;
3809}
3810
3811/*
3812 * Should be used after trace_array_get(), trace_types_lock
3813 * ensures that i_cdev was already initialized.
3814 */
3815static inline int tracing_get_cpu(struct inode *inode)
3816{
3817	if (inode->i_cdev) /* See trace_create_cpu_file() */
3818		return (long)inode->i_cdev - 1;
3819	return RING_BUFFER_ALL_CPUS;
3820}
3821
3822static const struct seq_operations tracer_seq_ops = {
3823	.start		= s_start,
3824	.next		= s_next,
3825	.stop		= s_stop,
3826	.show		= s_show,
3827};
3828
3829static struct trace_iterator *
3830__tracing_open(struct inode *inode, struct file *file, bool snapshot)
3831{
3832	struct trace_array *tr = inode->i_private;
3833	struct trace_iterator *iter;
3834	int cpu;
3835
3836	if (tracing_disabled)
3837		return ERR_PTR(-ENODEV);
3838
3839	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3840	if (!iter)
3841		return ERR_PTR(-ENOMEM);
3842
3843	iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3844				    GFP_KERNEL);
3845	if (!iter->buffer_iter)
3846		goto release;
3847
3848	/*
3849	 * We make a copy of the current tracer to avoid concurrent
3850	 * changes on it while we are reading.
3851	 */
3852	mutex_lock(&trace_types_lock);
3853	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3854	if (!iter->trace)
3855		goto fail;
3856
3857	*iter->trace = *tr->current_trace;
3858
3859	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3860		goto fail;
3861
3862	iter->tr = tr;
3863
3864#ifdef CONFIG_TRACER_MAX_TRACE
3865	/* Currently only the top directory has a snapshot */
3866	if (tr->current_trace->print_max || snapshot)
3867		iter->trace_buffer = &tr->max_buffer;
3868	else
3869#endif
3870		iter->trace_buffer = &tr->trace_buffer;
3871	iter->snapshot = snapshot;
3872	iter->pos = -1;
3873	iter->cpu_file = tracing_get_cpu(inode);
3874	mutex_init(&iter->mutex);
3875
3876	/* Notify the tracer early; before we stop tracing. */
3877	if (iter->trace && iter->trace->open)
3878		iter->trace->open(iter);
3879
3880	/* Annotate start of buffers if we had overruns */
3881	if (ring_buffer_overruns(iter->trace_buffer->buffer))
3882		iter->iter_flags |= TRACE_FILE_ANNOTATE;
3883
3884	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
3885	if (trace_clocks[tr->clock_id].in_ns)
3886		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3887
3888	/* stop the trace while dumping if we are not opening "snapshot" */
3889	if (!iter->snapshot)
3890		tracing_stop_tr(tr);
3891
3892	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3893		for_each_tracing_cpu(cpu) {
3894			iter->buffer_iter[cpu] =
3895				ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3896		}
3897		ring_buffer_read_prepare_sync();
3898		for_each_tracing_cpu(cpu) {
3899			ring_buffer_read_start(iter->buffer_iter[cpu]);
3900			tracing_iter_reset(iter, cpu);
3901		}
3902	} else {
3903		cpu = iter->cpu_file;
3904		iter->buffer_iter[cpu] =
3905			ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3906		ring_buffer_read_prepare_sync();
3907		ring_buffer_read_start(iter->buffer_iter[cpu]);
3908		tracing_iter_reset(iter, cpu);
3909	}
3910
3911	mutex_unlock(&trace_types_lock);
3912
3913	return iter;
3914
3915 fail:
3916	mutex_unlock(&trace_types_lock);
3917	kfree(iter->trace);
3918	kfree(iter->buffer_iter);
3919release:
3920	seq_release_private(inode, file);
3921	return ERR_PTR(-ENOMEM);
3922}
3923
3924int tracing_open_generic(struct inode *inode, struct file *filp)
3925{
3926	if (tracing_disabled)
3927		return -ENODEV;
3928
3929	filp->private_data = inode->i_private;
3930	return 0;
3931}
3932
3933bool tracing_is_disabled(void)
3934{
3935	return (tracing_disabled) ? true: false;
3936}
3937
3938/*
3939 * Open and update trace_array ref count.
3940 * Must have the current trace_array passed to it.
3941 */
3942static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3943{
3944	struct trace_array *tr = inode->i_private;
3945
3946	if (tracing_disabled)
3947		return -ENODEV;
3948
3949	if (trace_array_get(tr) < 0)
3950		return -ENODEV;
3951
3952	filp->private_data = inode->i_private;
3953
3954	return 0;
3955}
3956
3957static int tracing_release(struct inode *inode, struct file *file)
3958{
3959	struct trace_array *tr = inode->i_private;
3960	struct seq_file *m = file->private_data;
3961	struct trace_iterator *iter;
3962	int cpu;
3963
3964	if (!(file->f_mode & FMODE_READ)) {
3965		trace_array_put(tr);
3966		return 0;
3967	}
3968
3969	/* Writes do not use seq_file */
3970	iter = m->private;
3971	mutex_lock(&trace_types_lock);
3972
3973	for_each_tracing_cpu(cpu) {
3974		if (iter->buffer_iter[cpu])
3975			ring_buffer_read_finish(iter->buffer_iter[cpu]);
3976	}
3977
3978	if (iter->trace && iter->trace->close)
3979		iter->trace->close(iter);
3980
3981	if (!iter->snapshot)
3982		/* reenable tracing if it was previously enabled */
3983		tracing_start_tr(tr);
3984
3985	__trace_array_put(tr);
3986
3987	mutex_unlock(&trace_types_lock);
3988
3989	mutex_destroy(&iter->mutex);
3990	free_cpumask_var(iter->started);
3991	kfree(iter->trace);
3992	kfree(iter->buffer_iter);
3993	seq_release_private(inode, file);
3994
3995	return 0;
3996}
3997
3998static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3999{
4000	struct trace_array *tr = inode->i_private;
4001
4002	trace_array_put(tr);
4003	return 0;
4004}
4005
4006static int tracing_single_release_tr(struct inode *inode, struct file *file)
4007{
4008	struct trace_array *tr = inode->i_private;
4009
4010	trace_array_put(tr);
4011
4012	return single_release(inode, file);
4013}
4014
4015static int tracing_open(struct inode *inode, struct file *file)
4016{
4017	struct trace_array *tr = inode->i_private;
4018	struct trace_iterator *iter;
4019	int ret = 0;
4020
4021	if (trace_array_get(tr) < 0)
4022		return -ENODEV;
4023
4024	/* If this file was open for write, then erase contents */
4025	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4026		int cpu = tracing_get_cpu(inode);
4027		struct trace_buffer *trace_buf = &tr->trace_buffer;
4028
4029#ifdef CONFIG_TRACER_MAX_TRACE
4030		if (tr->current_trace->print_max)
4031			trace_buf = &tr->max_buffer;
4032#endif
4033
4034		if (cpu == RING_BUFFER_ALL_CPUS)
4035			tracing_reset_online_cpus(trace_buf);
4036		else
4037			tracing_reset(trace_buf, cpu);
4038	}
4039
4040	if (file->f_mode & FMODE_READ) {
4041		iter = __tracing_open(inode, file, false);
4042		if (IS_ERR(iter))
4043			ret = PTR_ERR(iter);
4044		else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4045			iter->iter_flags |= TRACE_FILE_LAT_FMT;
4046	}
4047
4048	if (ret < 0)
4049		trace_array_put(tr);
4050
4051	return ret;
4052}
4053
4054/*
4055 * Some tracers are not suitable for instance buffers.
4056 * A tracer is always available for the global array (toplevel)
4057 * or if it explicitly states that it is.
4058 */
4059static bool
4060trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4061{
4062	return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4063}
4064
4065/* Find the next tracer that this trace array may use */
4066static struct tracer *
4067get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4068{
4069	while (t && !trace_ok_for_array(t, tr))
4070		t = t->next;
4071
4072	return t;
4073}
4074
4075static void *
4076t_next(struct seq_file *m, void *v, loff_t *pos)
4077{
4078	struct trace_array *tr = m->private;
4079	struct tracer *t = v;
4080
4081	(*pos)++;
4082
4083	if (t)
4084		t = get_tracer_for_array(tr, t->next);
4085
4086	return t;
4087}
4088
4089static void *t_start(struct seq_file *m, loff_t *pos)
4090{
4091	struct trace_array *tr = m->private;
4092	struct tracer *t;
4093	loff_t l = 0;
4094
4095	mutex_lock(&trace_types_lock);
4096
4097	t = get_tracer_for_array(tr, trace_types);
4098	for (; t && l < *pos; t = t_next(m, t, &l))
4099			;
4100
4101	return t;
4102}
4103
4104static void t_stop(struct seq_file *m, void *p)
4105{
4106	mutex_unlock(&trace_types_lock);
4107}
4108
4109static int t_show(struct seq_file *m, void *v)
4110{
4111	struct tracer *t = v;
4112
4113	if (!t)
4114		return 0;
4115
4116	seq_puts(m, t->name);
4117	if (t->next)
4118		seq_putc(m, ' ');
4119	else
4120		seq_putc(m, '\n');
4121
4122	return 0;
4123}
4124
4125static const struct seq_operations show_traces_seq_ops = {
4126	.start		= t_start,
4127	.next		= t_next,
4128	.stop		= t_stop,
4129	.show		= t_show,
4130};
4131
4132static int show_traces_open(struct inode *inode, struct file *file)
4133{
4134	struct trace_array *tr = inode->i_private;
4135	struct seq_file *m;
4136	int ret;
4137
4138	if (tracing_disabled)
4139		return -ENODEV;
4140
4141	ret = seq_open(file, &show_traces_seq_ops);
4142	if (ret)
4143		return ret;
4144
4145	m = file->private_data;
4146	m->private = tr;
4147
4148	return 0;
4149}
4150
4151static ssize_t
4152tracing_write_stub(struct file *filp, const char __user *ubuf,
4153		   size_t count, loff_t *ppos)
4154{
4155	return count;
4156}
4157
4158loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4159{
4160	int ret;
4161
4162	if (file->f_mode & FMODE_READ)
4163		ret = seq_lseek(file, offset, whence);
4164	else
4165		file->f_pos = ret = 0;
4166
4167	return ret;
4168}
4169
4170static const struct file_operations tracing_fops = {
4171	.open		= tracing_open,
4172	.read		= seq_read,
4173	.write		= tracing_write_stub,
4174	.llseek		= tracing_lseek,
4175	.release	= tracing_release,
4176};
4177
4178static const struct file_operations show_traces_fops = {
4179	.open		= show_traces_open,
4180	.read		= seq_read,
4181	.release	= seq_release,
4182	.llseek		= seq_lseek,
4183};
4184
 
 
 
 
 
 
 
 
 
 
 
 
4185static ssize_t
4186tracing_cpumask_read(struct file *filp, char __user *ubuf,
4187		     size_t count, loff_t *ppos)
4188{
4189	struct trace_array *tr = file_inode(filp)->i_private;
4190	char *mask_str;
4191	int len;
4192
4193	len = snprintf(NULL, 0, "%*pb\n",
4194		       cpumask_pr_args(tr->tracing_cpumask)) + 1;
4195	mask_str = kmalloc(len, GFP_KERNEL);
4196	if (!mask_str)
4197		return -ENOMEM;
4198
4199	len = snprintf(mask_str, len, "%*pb\n",
4200		       cpumask_pr_args(tr->tracing_cpumask));
4201	if (len >= count) {
4202		count = -EINVAL;
4203		goto out_err;
4204	}
4205	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4206
4207out_err:
4208	kfree(mask_str);
4209
4210	return count;
4211}
4212
4213static ssize_t
4214tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4215		      size_t count, loff_t *ppos)
4216{
4217	struct trace_array *tr = file_inode(filp)->i_private;
4218	cpumask_var_t tracing_cpumask_new;
4219	int err, cpu;
4220
4221	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4222		return -ENOMEM;
4223
4224	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4225	if (err)
4226		goto err_unlock;
4227
 
 
4228	local_irq_disable();
4229	arch_spin_lock(&tr->max_lock);
4230	for_each_tracing_cpu(cpu) {
4231		/*
4232		 * Increase/decrease the disabled counter if we are
4233		 * about to flip a bit in the cpumask:
4234		 */
4235		if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4236				!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4237			atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4238			ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
4239		}
4240		if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4241				cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4242			atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4243			ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
4244		}
4245	}
4246	arch_spin_unlock(&tr->max_lock);
4247	local_irq_enable();
4248
4249	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
 
 
4250	free_cpumask_var(tracing_cpumask_new);
4251
4252	return count;
4253
4254err_unlock:
4255	free_cpumask_var(tracing_cpumask_new);
4256
4257	return err;
4258}
4259
4260static const struct file_operations tracing_cpumask_fops = {
4261	.open		= tracing_open_generic_tr,
4262	.read		= tracing_cpumask_read,
4263	.write		= tracing_cpumask_write,
4264	.release	= tracing_release_generic_tr,
4265	.llseek		= generic_file_llseek,
4266};
4267
4268static int tracing_trace_options_show(struct seq_file *m, void *v)
4269{
4270	struct tracer_opt *trace_opts;
4271	struct trace_array *tr = m->private;
4272	u32 tracer_flags;
4273	int i;
4274
4275	mutex_lock(&trace_types_lock);
4276	tracer_flags = tr->current_trace->flags->val;
4277	trace_opts = tr->current_trace->flags->opts;
4278
4279	for (i = 0; trace_options[i]; i++) {
4280		if (tr->trace_flags & (1 << i))
4281			seq_printf(m, "%s\n", trace_options[i]);
4282		else
4283			seq_printf(m, "no%s\n", trace_options[i]);
4284	}
4285
4286	for (i = 0; trace_opts[i].name; i++) {
4287		if (tracer_flags & trace_opts[i].bit)
4288			seq_printf(m, "%s\n", trace_opts[i].name);
4289		else
4290			seq_printf(m, "no%s\n", trace_opts[i].name);
4291	}
4292	mutex_unlock(&trace_types_lock);
4293
4294	return 0;
4295}
4296
4297static int __set_tracer_option(struct trace_array *tr,
4298			       struct tracer_flags *tracer_flags,
4299			       struct tracer_opt *opts, int neg)
4300{
4301	struct tracer *trace = tracer_flags->trace;
4302	int ret;
4303
4304	ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4305	if (ret)
4306		return ret;
4307
4308	if (neg)
4309		tracer_flags->val &= ~opts->bit;
4310	else
4311		tracer_flags->val |= opts->bit;
4312	return 0;
4313}
4314
4315/* Try to assign a tracer specific option */
4316static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4317{
4318	struct tracer *trace = tr->current_trace;
4319	struct tracer_flags *tracer_flags = trace->flags;
4320	struct tracer_opt *opts = NULL;
4321	int i;
4322
4323	for (i = 0; tracer_flags->opts[i].name; i++) {
4324		opts = &tracer_flags->opts[i];
4325
4326		if (strcmp(cmp, opts->name) == 0)
4327			return __set_tracer_option(tr, trace->flags, opts, neg);
4328	}
4329
4330	return -EINVAL;
4331}
4332
4333/* Some tracers require overwrite to stay enabled */
4334int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4335{
4336	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4337		return -1;
4338
4339	return 0;
4340}
4341
4342int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4343{
4344	/* do nothing if flag is already set */
4345	if (!!(tr->trace_flags & mask) == !!enabled)
4346		return 0;
4347
4348	/* Give the tracer a chance to approve the change */
4349	if (tr->current_trace->flag_changed)
4350		if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4351			return -EINVAL;
4352
4353	if (enabled)
4354		tr->trace_flags |= mask;
4355	else
4356		tr->trace_flags &= ~mask;
4357
4358	if (mask == TRACE_ITER_RECORD_CMD)
4359		trace_event_enable_cmd_record(enabled);
4360
4361	if (mask == TRACE_ITER_RECORD_TGID) {
4362		if (!tgid_map)
4363			tgid_map = kzalloc((PID_MAX_DEFAULT + 1) * sizeof(*tgid_map),
4364					   GFP_KERNEL);
4365		if (!tgid_map) {
4366			tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4367			return -ENOMEM;
4368		}
4369
4370		trace_event_enable_tgid_record(enabled);
4371	}
4372
4373	if (mask == TRACE_ITER_EVENT_FORK)
4374		trace_event_follow_fork(tr, enabled);
4375
4376	if (mask == TRACE_ITER_FUNC_FORK)
4377		ftrace_pid_follow_fork(tr, enabled);
4378
4379	if (mask == TRACE_ITER_OVERWRITE) {
4380		ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
4381#ifdef CONFIG_TRACER_MAX_TRACE
4382		ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4383#endif
4384	}
4385
4386	if (mask == TRACE_ITER_PRINTK) {
4387		trace_printk_start_stop_comm(enabled);
4388		trace_printk_control(enabled);
4389	}
4390
4391	return 0;
4392}
4393
4394static int trace_set_options(struct trace_array *tr, char *option)
4395{
4396	char *cmp;
4397	int neg = 0;
4398	int ret = -ENODEV;
4399	int i;
4400	size_t orig_len = strlen(option);
4401
4402	cmp = strstrip(option);
4403
4404	if (strncmp(cmp, "no", 2) == 0) {
4405		neg = 1;
4406		cmp += 2;
4407	}
4408
4409	mutex_lock(&trace_types_lock);
4410
4411	for (i = 0; trace_options[i]; i++) {
4412		if (strcmp(cmp, trace_options[i]) == 0) {
4413			ret = set_tracer_flag(tr, 1 << i, !neg);
4414			break;
4415		}
4416	}
4417
4418	/* If no option could be set, test the specific tracer options */
4419	if (!trace_options[i])
4420		ret = set_tracer_option(tr, cmp, neg);
4421
4422	mutex_unlock(&trace_types_lock);
4423
4424	/*
4425	 * If the first trailing whitespace is replaced with '\0' by strstrip,
4426	 * turn it back into a space.
4427	 */
4428	if (orig_len > strlen(option))
4429		option[strlen(option)] = ' ';
4430
4431	return ret;
4432}
4433
4434static void __init apply_trace_boot_options(void)
4435{
4436	char *buf = trace_boot_options_buf;
4437	char *option;
4438
4439	while (true) {
4440		option = strsep(&buf, ",");
4441
4442		if (!option)
4443			break;
4444
4445		if (*option)
4446			trace_set_options(&global_trace, option);
4447
4448		/* Put back the comma to allow this to be called again */
4449		if (buf)
4450			*(buf - 1) = ',';
4451	}
4452}
4453
4454static ssize_t
4455tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4456			size_t cnt, loff_t *ppos)
4457{
4458	struct seq_file *m = filp->private_data;
4459	struct trace_array *tr = m->private;
4460	char buf[64];
4461	int ret;
4462
4463	if (cnt >= sizeof(buf))
4464		return -EINVAL;
4465
4466	if (copy_from_user(buf, ubuf, cnt))
4467		return -EFAULT;
4468
4469	buf[cnt] = 0;
4470
4471	ret = trace_set_options(tr, buf);
4472	if (ret < 0)
4473		return ret;
4474
4475	*ppos += cnt;
4476
4477	return cnt;
4478}
4479
4480static int tracing_trace_options_open(struct inode *inode, struct file *file)
4481{
4482	struct trace_array *tr = inode->i_private;
4483	int ret;
4484
4485	if (tracing_disabled)
4486		return -ENODEV;
4487
4488	if (trace_array_get(tr) < 0)
4489		return -ENODEV;
4490
4491	ret = single_open(file, tracing_trace_options_show, inode->i_private);
4492	if (ret < 0)
4493		trace_array_put(tr);
4494
4495	return ret;
4496}
4497
4498static const struct file_operations tracing_iter_fops = {
4499	.open		= tracing_trace_options_open,
4500	.read		= seq_read,
4501	.llseek		= seq_lseek,
4502	.release	= tracing_single_release_tr,
4503	.write		= tracing_trace_options_write,
4504};
4505
4506static const char readme_msg[] =
4507	"tracing mini-HOWTO:\n\n"
4508	"# echo 0 > tracing_on : quick way to disable tracing\n"
4509	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4510	" Important files:\n"
4511	"  trace\t\t\t- The static contents of the buffer\n"
4512	"\t\t\t  To clear the buffer write into this file: echo > trace\n"
4513	"  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4514	"  current_tracer\t- function and latency tracers\n"
4515	"  available_tracers\t- list of configured tracers for current_tracer\n"
4516	"  buffer_size_kb\t- view and modify size of per cpu buffer\n"
4517	"  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
4518	"  trace_clock\t\t-change the clock used to order events\n"
4519	"       local:   Per cpu clock but may not be synced across CPUs\n"
4520	"      global:   Synced across CPUs but slows tracing down.\n"
4521	"     counter:   Not a clock, but just an increment\n"
4522	"      uptime:   Jiffy counter from time of boot\n"
4523	"        perf:   Same clock that perf events use\n"
4524#ifdef CONFIG_X86_64
4525	"     x86-tsc:   TSC cycle counter\n"
4526#endif
4527	"\n  timestamp_mode\t-view the mode used to timestamp events\n"
4528	"       delta:   Delta difference against a buffer-wide timestamp\n"
4529	"    absolute:   Absolute (standalone) timestamp\n"
4530	"\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4531	"\n  trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4532	"  tracing_cpumask\t- Limit which CPUs to trace\n"
4533	"  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4534	"\t\t\t  Remove sub-buffer with rmdir\n"
4535	"  trace_options\t\t- Set format or modify how tracing happens\n"
4536	"\t\t\t  Disable an option by adding a suffix 'no' to the\n"
4537	"\t\t\t  option name\n"
4538	"  saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4539#ifdef CONFIG_DYNAMIC_FTRACE
4540	"\n  available_filter_functions - list of functions that can be filtered on\n"
4541	"  set_ftrace_filter\t- echo function name in here to only trace these\n"
4542	"\t\t\t  functions\n"
4543	"\t     accepts: func_full_name or glob-matching-pattern\n"
4544	"\t     modules: Can select a group via module\n"
4545	"\t      Format: :mod:<module-name>\n"
4546	"\t     example: echo :mod:ext3 > set_ftrace_filter\n"
4547	"\t    triggers: a command to perform when function is hit\n"
4548	"\t      Format: <function>:<trigger>[:count]\n"
4549	"\t     trigger: traceon, traceoff\n"
4550	"\t\t      enable_event:<system>:<event>\n"
4551	"\t\t      disable_event:<system>:<event>\n"
4552#ifdef CONFIG_STACKTRACE
4553	"\t\t      stacktrace\n"
4554#endif
4555#ifdef CONFIG_TRACER_SNAPSHOT
4556	"\t\t      snapshot\n"
4557#endif
4558	"\t\t      dump\n"
4559	"\t\t      cpudump\n"
4560	"\t     example: echo do_fault:traceoff > set_ftrace_filter\n"
4561	"\t              echo do_trap:traceoff:3 > set_ftrace_filter\n"
4562	"\t     The first one will disable tracing every time do_fault is hit\n"
4563	"\t     The second will disable tracing at most 3 times when do_trap is hit\n"
4564	"\t       The first time do trap is hit and it disables tracing, the\n"
4565	"\t       counter will decrement to 2. If tracing is already disabled,\n"
4566	"\t       the counter will not decrement. It only decrements when the\n"
4567	"\t       trigger did work\n"
4568	"\t     To remove trigger without count:\n"
4569	"\t       echo '!<function>:<trigger> > set_ftrace_filter\n"
4570	"\t     To remove trigger with a count:\n"
4571	"\t       echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4572	"  set_ftrace_notrace\t- echo function name in here to never trace.\n"
4573	"\t    accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4574	"\t    modules: Can select a group via module command :mod:\n"
4575	"\t    Does not accept triggers\n"
4576#endif /* CONFIG_DYNAMIC_FTRACE */
4577#ifdef CONFIG_FUNCTION_TRACER
4578	"  set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4579	"\t\t    (function)\n"
4580#endif
4581#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4582	"  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4583	"  set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4584	"  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4585#endif
4586#ifdef CONFIG_TRACER_SNAPSHOT
4587	"\n  snapshot\t\t- Like 'trace' but shows the content of the static\n"
4588	"\t\t\t  snapshot buffer. Read the contents for more\n"
4589	"\t\t\t  information\n"
4590#endif
4591#ifdef CONFIG_STACK_TRACER
4592	"  stack_trace\t\t- Shows the max stack trace when active\n"
4593	"  stack_max_size\t- Shows current max stack size that was traced\n"
4594	"\t\t\t  Write into this file to reset the max size (trigger a\n"
4595	"\t\t\t  new trace)\n"
4596#ifdef CONFIG_DYNAMIC_FTRACE
4597	"  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4598	"\t\t\t  traces\n"
4599#endif
4600#endif /* CONFIG_STACK_TRACER */
4601#ifdef CONFIG_KPROBE_EVENTS
4602	"  kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4603	"\t\t\t  Write into this file to define/undefine new trace events.\n"
4604#endif
4605#ifdef CONFIG_UPROBE_EVENTS
4606	"  uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4607	"\t\t\t  Write into this file to define/undefine new trace events.\n"
4608#endif
4609#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
4610	"\t  accepts: event-definitions (one definition per line)\n"
4611	"\t   Format: p[:[<group>/]<event>] <place> [<args>]\n"
4612	"\t           r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
4613	"\t           -:[<group>/]<event>\n"
4614#ifdef CONFIG_KPROBE_EVENTS
4615	"\t    place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4616  "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4617#endif
4618#ifdef CONFIG_UPROBE_EVENTS
4619	"\t    place: <path>:<offset>\n"
4620#endif
4621	"\t     args: <name>=fetcharg[:type]\n"
4622	"\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4623	"\t           $stack<index>, $stack, $retval, $comm\n"
4624	"\t     type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4625	"\t           b<bit-width>@<bit-offset>/<container-size>\n"
4626#endif
4627	"  events/\t\t- Directory containing all trace event subsystems:\n"
4628	"      enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4629	"  events/<system>/\t- Directory containing all trace events for <system>:\n"
4630	"      enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4631	"\t\t\t  events\n"
4632	"      filter\t\t- If set, only events passing filter are traced\n"
4633	"  events/<system>/<event>/\t- Directory containing control files for\n"
4634	"\t\t\t  <event>:\n"
4635	"      enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4636	"      filter\t\t- If set, only events passing filter are traced\n"
4637	"      trigger\t\t- If set, a command to perform when event is hit\n"
4638	"\t    Format: <trigger>[:count][if <filter>]\n"
4639	"\t   trigger: traceon, traceoff\n"
4640	"\t            enable_event:<system>:<event>\n"
4641	"\t            disable_event:<system>:<event>\n"
4642#ifdef CONFIG_HIST_TRIGGERS
4643	"\t            enable_hist:<system>:<event>\n"
4644	"\t            disable_hist:<system>:<event>\n"
4645#endif
4646#ifdef CONFIG_STACKTRACE
4647	"\t\t    stacktrace\n"
4648#endif
4649#ifdef CONFIG_TRACER_SNAPSHOT
4650	"\t\t    snapshot\n"
4651#endif
4652#ifdef CONFIG_HIST_TRIGGERS
4653	"\t\t    hist (see below)\n"
4654#endif
4655	"\t   example: echo traceoff > events/block/block_unplug/trigger\n"
4656	"\t            echo traceoff:3 > events/block/block_unplug/trigger\n"
4657	"\t            echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4658	"\t                  events/block/block_unplug/trigger\n"
4659	"\t   The first disables tracing every time block_unplug is hit.\n"
4660	"\t   The second disables tracing the first 3 times block_unplug is hit.\n"
4661	"\t   The third enables the kmalloc event the first 3 times block_unplug\n"
4662	"\t     is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4663	"\t   Like function triggers, the counter is only decremented if it\n"
4664	"\t    enabled or disabled tracing.\n"
4665	"\t   To remove a trigger without a count:\n"
4666	"\t     echo '!<trigger> > <system>/<event>/trigger\n"
4667	"\t   To remove a trigger with a count:\n"
4668	"\t     echo '!<trigger>:0 > <system>/<event>/trigger\n"
4669	"\t   Filters can be ignored when removing a trigger.\n"
4670#ifdef CONFIG_HIST_TRIGGERS
4671	"      hist trigger\t- If set, event hits are aggregated into a hash table\n"
4672	"\t    Format: hist:keys=<field1[,field2,...]>\n"
4673	"\t            [:values=<field1[,field2,...]>]\n"
4674	"\t            [:sort=<field1[,field2,...]>]\n"
4675	"\t            [:size=#entries]\n"
4676	"\t            [:pause][:continue][:clear]\n"
4677	"\t            [:name=histname1]\n"
4678	"\t            [if <filter>]\n\n"
4679	"\t    When a matching event is hit, an entry is added to a hash\n"
4680	"\t    table using the key(s) and value(s) named, and the value of a\n"
4681	"\t    sum called 'hitcount' is incremented.  Keys and values\n"
4682	"\t    correspond to fields in the event's format description.  Keys\n"
4683	"\t    can be any field, or the special string 'stacktrace'.\n"
4684	"\t    Compound keys consisting of up to two fields can be specified\n"
4685	"\t    by the 'keys' keyword.  Values must correspond to numeric\n"
4686	"\t    fields.  Sort keys consisting of up to two fields can be\n"
4687	"\t    specified using the 'sort' keyword.  The sort direction can\n"
4688	"\t    be modified by appending '.descending' or '.ascending' to a\n"
4689	"\t    sort field.  The 'size' parameter can be used to specify more\n"
4690	"\t    or fewer than the default 2048 entries for the hashtable size.\n"
4691	"\t    If a hist trigger is given a name using the 'name' parameter,\n"
4692	"\t    its histogram data will be shared with other triggers of the\n"
4693	"\t    same name, and trigger hits will update this common data.\n\n"
4694	"\t    Reading the 'hist' file for the event will dump the hash\n"
4695	"\t    table in its entirety to stdout.  If there are multiple hist\n"
4696	"\t    triggers attached to an event, there will be a table for each\n"
4697	"\t    trigger in the output.  The table displayed for a named\n"
4698	"\t    trigger will be the same as any other instance having the\n"
4699	"\t    same name.  The default format used to display a given field\n"
4700	"\t    can be modified by appending any of the following modifiers\n"
4701	"\t    to the field name, as applicable:\n\n"
4702	"\t            .hex        display a number as a hex value\n"
4703	"\t            .sym        display an address as a symbol\n"
4704	"\t            .sym-offset display an address as a symbol and offset\n"
4705	"\t            .execname   display a common_pid as a program name\n"
4706	"\t            .syscall    display a syscall id as a syscall name\n"
4707	"\t            .log2       display log2 value rather than raw number\n"
4708	"\t            .usecs      display a common_timestamp in microseconds\n\n"
4709	"\t    The 'pause' parameter can be used to pause an existing hist\n"
4710	"\t    trigger or to start a hist trigger but not log any events\n"
4711	"\t    until told to do so.  'continue' can be used to start or\n"
4712	"\t    restart a paused hist trigger.\n\n"
4713	"\t    The 'clear' parameter will clear the contents of a running\n"
4714	"\t    hist trigger and leave its current paused/active state\n"
4715	"\t    unchanged.\n\n"
4716	"\t    The enable_hist and disable_hist triggers can be used to\n"
4717	"\t    have one event conditionally start and stop another event's\n"
4718	"\t    already-attached hist trigger.  The syntax is analagous to\n"
4719	"\t    the enable_event and disable_event triggers.\n"
4720#endif
4721;
4722
4723static ssize_t
4724tracing_readme_read(struct file *filp, char __user *ubuf,
4725		       size_t cnt, loff_t *ppos)
4726{
4727	return simple_read_from_buffer(ubuf, cnt, ppos,
4728					readme_msg, strlen(readme_msg));
4729}
4730
4731static const struct file_operations tracing_readme_fops = {
4732	.open		= tracing_open_generic,
4733	.read		= tracing_readme_read,
4734	.llseek		= generic_file_llseek,
4735};
4736
4737static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
4738{
4739	int *ptr = v;
4740
4741	if (*pos || m->count)
4742		ptr++;
4743
4744	(*pos)++;
4745
4746	for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
4747		if (trace_find_tgid(*ptr))
4748			return ptr;
4749	}
4750
4751	return NULL;
4752}
4753
4754static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
4755{
4756	void *v;
4757	loff_t l = 0;
4758
4759	if (!tgid_map)
4760		return NULL;
4761
4762	v = &tgid_map[0];
4763	while (l <= *pos) {
4764		v = saved_tgids_next(m, v, &l);
4765		if (!v)
4766			return NULL;
4767	}
4768
4769	return v;
4770}
4771
4772static void saved_tgids_stop(struct seq_file *m, void *v)
4773{
4774}
4775
4776static int saved_tgids_show(struct seq_file *m, void *v)
4777{
4778	int pid = (int *)v - tgid_map;
4779
4780	seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
4781	return 0;
4782}
4783
4784static const struct seq_operations tracing_saved_tgids_seq_ops = {
4785	.start		= saved_tgids_start,
4786	.stop		= saved_tgids_stop,
4787	.next		= saved_tgids_next,
4788	.show		= saved_tgids_show,
4789};
4790
4791static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
4792{
4793	if (tracing_disabled)
4794		return -ENODEV;
4795
4796	return seq_open(filp, &tracing_saved_tgids_seq_ops);
4797}
4798
4799
4800static const struct file_operations tracing_saved_tgids_fops = {
4801	.open		= tracing_saved_tgids_open,
4802	.read		= seq_read,
4803	.llseek		= seq_lseek,
4804	.release	= seq_release,
4805};
4806
4807static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4808{
4809	unsigned int *ptr = v;
4810
4811	if (*pos || m->count)
4812		ptr++;
4813
4814	(*pos)++;
4815
4816	for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4817	     ptr++) {
4818		if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4819			continue;
4820
4821		return ptr;
4822	}
4823
4824	return NULL;
4825}
4826
4827static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4828{
4829	void *v;
4830	loff_t l = 0;
4831
4832	preempt_disable();
4833	arch_spin_lock(&trace_cmdline_lock);
4834
4835	v = &savedcmd->map_cmdline_to_pid[0];
4836	while (l <= *pos) {
4837		v = saved_cmdlines_next(m, v, &l);
4838		if (!v)
4839			return NULL;
4840	}
4841
4842	return v;
4843}
4844
4845static void saved_cmdlines_stop(struct seq_file *m, void *v)
4846{
4847	arch_spin_unlock(&trace_cmdline_lock);
4848	preempt_enable();
4849}
4850
4851static int saved_cmdlines_show(struct seq_file *m, void *v)
4852{
4853	char buf[TASK_COMM_LEN];
4854	unsigned int *pid = v;
4855
4856	__trace_find_cmdline(*pid, buf);
4857	seq_printf(m, "%d %s\n", *pid, buf);
4858	return 0;
4859}
4860
4861static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4862	.start		= saved_cmdlines_start,
4863	.next		= saved_cmdlines_next,
4864	.stop		= saved_cmdlines_stop,
4865	.show		= saved_cmdlines_show,
4866};
4867
4868static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4869{
4870	if (tracing_disabled)
4871		return -ENODEV;
4872
4873	return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
4874}
4875
4876static const struct file_operations tracing_saved_cmdlines_fops = {
4877	.open		= tracing_saved_cmdlines_open,
4878	.read		= seq_read,
4879	.llseek		= seq_lseek,
4880	.release	= seq_release,
4881};
4882
4883static ssize_t
4884tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4885				 size_t cnt, loff_t *ppos)
4886{
4887	char buf[64];
4888	int r;
4889
4890	arch_spin_lock(&trace_cmdline_lock);
4891	r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
4892	arch_spin_unlock(&trace_cmdline_lock);
4893
4894	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4895}
4896
4897static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4898{
4899	kfree(s->saved_cmdlines);
4900	kfree(s->map_cmdline_to_pid);
4901	kfree(s);
4902}
4903
4904static int tracing_resize_saved_cmdlines(unsigned int val)
4905{
4906	struct saved_cmdlines_buffer *s, *savedcmd_temp;
4907
4908	s = kmalloc(sizeof(*s), GFP_KERNEL);
4909	if (!s)
4910		return -ENOMEM;
4911
4912	if (allocate_cmdlines_buffer(val, s) < 0) {
4913		kfree(s);
4914		return -ENOMEM;
4915	}
4916
4917	arch_spin_lock(&trace_cmdline_lock);
4918	savedcmd_temp = savedcmd;
4919	savedcmd = s;
4920	arch_spin_unlock(&trace_cmdline_lock);
4921	free_saved_cmdlines_buffer(savedcmd_temp);
4922
4923	return 0;
4924}
4925
4926static ssize_t
4927tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4928				  size_t cnt, loff_t *ppos)
4929{
4930	unsigned long val;
4931	int ret;
4932
4933	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4934	if (ret)
4935		return ret;
4936
4937	/* must have at least 1 entry or less than PID_MAX_DEFAULT */
4938	if (!val || val > PID_MAX_DEFAULT)
4939		return -EINVAL;
4940
4941	ret = tracing_resize_saved_cmdlines((unsigned int)val);
4942	if (ret < 0)
4943		return ret;
4944
4945	*ppos += cnt;
4946
4947	return cnt;
4948}
4949
4950static const struct file_operations tracing_saved_cmdlines_size_fops = {
4951	.open		= tracing_open_generic,
4952	.read		= tracing_saved_cmdlines_size_read,
4953	.write		= tracing_saved_cmdlines_size_write,
4954};
4955
4956#ifdef CONFIG_TRACE_EVAL_MAP_FILE
4957static union trace_eval_map_item *
4958update_eval_map(union trace_eval_map_item *ptr)
4959{
4960	if (!ptr->map.eval_string) {
4961		if (ptr->tail.next) {
4962			ptr = ptr->tail.next;
4963			/* Set ptr to the next real item (skip head) */
4964			ptr++;
4965		} else
4966			return NULL;
4967	}
4968	return ptr;
4969}
4970
4971static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
4972{
4973	union trace_eval_map_item *ptr = v;
4974
4975	/*
4976	 * Paranoid! If ptr points to end, we don't want to increment past it.
4977	 * This really should never happen.
4978	 */
4979	ptr = update_eval_map(ptr);
4980	if (WARN_ON_ONCE(!ptr))
4981		return NULL;
4982
4983	ptr++;
4984
4985	(*pos)++;
4986
4987	ptr = update_eval_map(ptr);
4988
4989	return ptr;
4990}
4991
4992static void *eval_map_start(struct seq_file *m, loff_t *pos)
4993{
4994	union trace_eval_map_item *v;
4995	loff_t l = 0;
4996
4997	mutex_lock(&trace_eval_mutex);
4998
4999	v = trace_eval_maps;
5000	if (v)
5001		v++;
5002
5003	while (v && l < *pos) {
5004		v = eval_map_next(m, v, &l);
5005	}
5006
5007	return v;
5008}
5009
5010static void eval_map_stop(struct seq_file *m, void *v)
5011{
5012	mutex_unlock(&trace_eval_mutex);
5013}
5014
5015static int eval_map_show(struct seq_file *m, void *v)
5016{
5017	union trace_eval_map_item *ptr = v;
5018
5019	seq_printf(m, "%s %ld (%s)\n",
5020		   ptr->map.eval_string, ptr->map.eval_value,
5021		   ptr->map.system);
5022
5023	return 0;
5024}
5025
5026static const struct seq_operations tracing_eval_map_seq_ops = {
5027	.start		= eval_map_start,
5028	.next		= eval_map_next,
5029	.stop		= eval_map_stop,
5030	.show		= eval_map_show,
5031};
5032
5033static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5034{
5035	if (tracing_disabled)
5036		return -ENODEV;
5037
5038	return seq_open(filp, &tracing_eval_map_seq_ops);
5039}
5040
5041static const struct file_operations tracing_eval_map_fops = {
5042	.open		= tracing_eval_map_open,
5043	.read		= seq_read,
5044	.llseek		= seq_lseek,
5045	.release	= seq_release,
5046};
5047
5048static inline union trace_eval_map_item *
5049trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5050{
5051	/* Return tail of array given the head */
5052	return ptr + ptr->head.length + 1;
5053}
5054
5055static void
5056trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5057			   int len)
5058{
5059	struct trace_eval_map **stop;
5060	struct trace_eval_map **map;
5061	union trace_eval_map_item *map_array;
5062	union trace_eval_map_item *ptr;
5063
5064	stop = start + len;
5065
5066	/*
5067	 * The trace_eval_maps contains the map plus a head and tail item,
5068	 * where the head holds the module and length of array, and the
5069	 * tail holds a pointer to the next list.
5070	 */
5071	map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
5072	if (!map_array) {
5073		pr_warn("Unable to allocate trace eval mapping\n");
5074		return;
5075	}
5076
5077	mutex_lock(&trace_eval_mutex);
5078
5079	if (!trace_eval_maps)
5080		trace_eval_maps = map_array;
5081	else {
5082		ptr = trace_eval_maps;
5083		for (;;) {
5084			ptr = trace_eval_jmp_to_tail(ptr);
5085			if (!ptr->tail.next)
5086				break;
5087			ptr = ptr->tail.next;
5088
5089		}
5090		ptr->tail.next = map_array;
5091	}
5092	map_array->head.mod = mod;
5093	map_array->head.length = len;
5094	map_array++;
5095
5096	for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5097		map_array->map = **map;
5098		map_array++;
5099	}
5100	memset(map_array, 0, sizeof(*map_array));
5101
5102	mutex_unlock(&trace_eval_mutex);
5103}
5104
5105static void trace_create_eval_file(struct dentry *d_tracer)
5106{
5107	trace_create_file("eval_map", 0444, d_tracer,
5108			  NULL, &tracing_eval_map_fops);
5109}
5110
5111#else /* CONFIG_TRACE_EVAL_MAP_FILE */
5112static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5113static inline void trace_insert_eval_map_file(struct module *mod,
5114			      struct trace_eval_map **start, int len) { }
5115#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5116
5117static void trace_insert_eval_map(struct module *mod,
5118				  struct trace_eval_map **start, int len)
5119{
5120	struct trace_eval_map **map;
5121
5122	if (len <= 0)
5123		return;
5124
5125	map = start;
5126
5127	trace_event_eval_update(map, len);
5128
5129	trace_insert_eval_map_file(mod, start, len);
5130}
5131
5132static ssize_t
5133tracing_set_trace_read(struct file *filp, char __user *ubuf,
5134		       size_t cnt, loff_t *ppos)
5135{
5136	struct trace_array *tr = filp->private_data;
5137	char buf[MAX_TRACER_SIZE+2];
5138	int r;
5139
5140	mutex_lock(&trace_types_lock);
5141	r = sprintf(buf, "%s\n", tr->current_trace->name);
5142	mutex_unlock(&trace_types_lock);
5143
5144	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5145}
5146
5147int tracer_init(struct tracer *t, struct trace_array *tr)
5148{
5149	tracing_reset_online_cpus(&tr->trace_buffer);
5150	return t->init(tr);
5151}
5152
5153static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
5154{
5155	int cpu;
5156
5157	for_each_tracing_cpu(cpu)
5158		per_cpu_ptr(buf->data, cpu)->entries = val;
5159}
5160
5161#ifdef CONFIG_TRACER_MAX_TRACE
5162/* resize @tr's buffer to the size of @size_tr's entries */
5163static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
5164					struct trace_buffer *size_buf, int cpu_id)
5165{
5166	int cpu, ret = 0;
5167
5168	if (cpu_id == RING_BUFFER_ALL_CPUS) {
5169		for_each_tracing_cpu(cpu) {
5170			ret = ring_buffer_resize(trace_buf->buffer,
5171				 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5172			if (ret < 0)
5173				break;
5174			per_cpu_ptr(trace_buf->data, cpu)->entries =
5175				per_cpu_ptr(size_buf->data, cpu)->entries;
5176		}
5177	} else {
5178		ret = ring_buffer_resize(trace_buf->buffer,
5179				 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5180		if (ret == 0)
5181			per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5182				per_cpu_ptr(size_buf->data, cpu_id)->entries;
5183	}
5184
5185	return ret;
5186}
5187#endif /* CONFIG_TRACER_MAX_TRACE */
5188
5189static int __tracing_resize_ring_buffer(struct trace_array *tr,
5190					unsigned long size, int cpu)
5191{
5192	int ret;
5193
5194	/*
5195	 * If kernel or user changes the size of the ring buffer
5196	 * we use the size that was given, and we can forget about
5197	 * expanding it later.
5198	 */
5199	ring_buffer_expanded = true;
5200
5201	/* May be called before buffers are initialized */
5202	if (!tr->trace_buffer.buffer)
5203		return 0;
5204
5205	ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
5206	if (ret < 0)
5207		return ret;
5208
5209#ifdef CONFIG_TRACER_MAX_TRACE
5210	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5211	    !tr->current_trace->use_max_tr)
5212		goto out;
5213
5214	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5215	if (ret < 0) {
5216		int r = resize_buffer_duplicate_size(&tr->trace_buffer,
5217						     &tr->trace_buffer, cpu);
5218		if (r < 0) {
5219			/*
5220			 * AARGH! We are left with different
5221			 * size max buffer!!!!
5222			 * The max buffer is our "snapshot" buffer.
5223			 * When a tracer needs a snapshot (one of the
5224			 * latency tracers), it swaps the max buffer
5225			 * with the saved snap shot. We succeeded to
5226			 * update the size of the main buffer, but failed to
5227			 * update the size of the max buffer. But when we tried
5228			 * to reset the main buffer to the original size, we
5229			 * failed there too. This is very unlikely to
5230			 * happen, but if it does, warn and kill all
5231			 * tracing.
5232			 */
5233			WARN_ON(1);
5234			tracing_disabled = 1;
5235		}
5236		return ret;
5237	}
5238
5239	if (cpu == RING_BUFFER_ALL_CPUS)
5240		set_buffer_entries(&tr->max_buffer, size);
5241	else
5242		per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5243
5244 out:
5245#endif /* CONFIG_TRACER_MAX_TRACE */
5246
5247	if (cpu == RING_BUFFER_ALL_CPUS)
5248		set_buffer_entries(&tr->trace_buffer, size);
5249	else
5250		per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
5251
5252	return ret;
5253}
5254
5255static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5256					  unsigned long size, int cpu_id)
5257{
5258	int ret = size;
5259
5260	mutex_lock(&trace_types_lock);
5261
5262	if (cpu_id != RING_BUFFER_ALL_CPUS) {
5263		/* make sure, this cpu is enabled in the mask */
5264		if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5265			ret = -EINVAL;
5266			goto out;
5267		}
5268	}
5269
5270	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5271	if (ret < 0)
5272		ret = -ENOMEM;
5273
5274out:
5275	mutex_unlock(&trace_types_lock);
5276
5277	return ret;
5278}
5279
5280
5281/**
5282 * tracing_update_buffers - used by tracing facility to expand ring buffers
5283 *
5284 * To save on memory when the tracing is never used on a system with it
5285 * configured in. The ring buffers are set to a minimum size. But once
5286 * a user starts to use the tracing facility, then they need to grow
5287 * to their default size.
5288 *
5289 * This function is to be called when a tracer is about to be used.
5290 */
5291int tracing_update_buffers(void)
5292{
5293	int ret = 0;
5294
5295	mutex_lock(&trace_types_lock);
5296	if (!ring_buffer_expanded)
5297		ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5298						RING_BUFFER_ALL_CPUS);
5299	mutex_unlock(&trace_types_lock);
5300
5301	return ret;
5302}
5303
5304struct trace_option_dentry;
5305
5306static void
5307create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5308
5309/*
5310 * Used to clear out the tracer before deletion of an instance.
5311 * Must have trace_types_lock held.
5312 */
5313static void tracing_set_nop(struct trace_array *tr)
5314{
5315	if (tr->current_trace == &nop_trace)
5316		return;
5317	
5318	tr->current_trace->enabled--;
5319
5320	if (tr->current_trace->reset)
5321		tr->current_trace->reset(tr);
5322
5323	tr->current_trace = &nop_trace;
5324}
5325
5326static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5327{
5328	/* Only enable if the directory has been created already. */
5329	if (!tr->dir)
5330		return;
5331
5332	create_trace_option_files(tr, t);
5333}
5334
5335static int tracing_set_tracer(struct trace_array *tr, const char *buf)
5336{
5337	struct tracer *t;
5338#ifdef CONFIG_TRACER_MAX_TRACE
5339	bool had_max_tr;
5340#endif
5341	int ret = 0;
5342
5343	mutex_lock(&trace_types_lock);
5344
5345	if (!ring_buffer_expanded) {
5346		ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5347						RING_BUFFER_ALL_CPUS);
5348		if (ret < 0)
5349			goto out;
5350		ret = 0;
5351	}
5352
5353	for (t = trace_types; t; t = t->next) {
5354		if (strcmp(t->name, buf) == 0)
5355			break;
5356	}
5357	if (!t) {
5358		ret = -EINVAL;
5359		goto out;
5360	}
5361	if (t == tr->current_trace)
5362		goto out;
5363
5364	/* Some tracers won't work on kernel command line */
5365	if (system_state < SYSTEM_RUNNING && t->noboot) {
5366		pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5367			t->name);
5368		goto out;
5369	}
5370
5371	/* Some tracers are only allowed for the top level buffer */
5372	if (!trace_ok_for_array(t, tr)) {
5373		ret = -EINVAL;
5374		goto out;
5375	}
5376
5377	/* If trace pipe files are being read, we can't change the tracer */
5378	if (tr->current_trace->ref) {
5379		ret = -EBUSY;
5380		goto out;
5381	}
5382
5383	trace_branch_disable();
5384
5385	tr->current_trace->enabled--;
5386
5387	if (tr->current_trace->reset)
5388		tr->current_trace->reset(tr);
5389
5390	/* Current trace needs to be nop_trace before synchronize_sched */
5391	tr->current_trace = &nop_trace;
5392
5393#ifdef CONFIG_TRACER_MAX_TRACE
5394	had_max_tr = tr->allocated_snapshot;
5395
5396	if (had_max_tr && !t->use_max_tr) {
5397		/*
5398		 * We need to make sure that the update_max_tr sees that
5399		 * current_trace changed to nop_trace to keep it from
5400		 * swapping the buffers after we resize it.
5401		 * The update_max_tr is called from interrupts disabled
5402		 * so a synchronized_sched() is sufficient.
5403		 */
5404		synchronize_sched();
5405		free_snapshot(tr);
5406	}
5407#endif
5408
5409#ifdef CONFIG_TRACER_MAX_TRACE
5410	if (t->use_max_tr && !had_max_tr) {
5411		ret = tracing_alloc_snapshot_instance(tr);
5412		if (ret < 0)
5413			goto out;
5414	}
5415#endif
5416
5417	if (t->init) {
5418		ret = tracer_init(t, tr);
5419		if (ret)
5420			goto out;
5421	}
5422
5423	tr->current_trace = t;
5424	tr->current_trace->enabled++;
5425	trace_branch_enable(tr);
5426 out:
5427	mutex_unlock(&trace_types_lock);
5428
5429	return ret;
5430}
5431
5432static ssize_t
5433tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5434			size_t cnt, loff_t *ppos)
5435{
5436	struct trace_array *tr = filp->private_data;
5437	char buf[MAX_TRACER_SIZE+1];
5438	int i;
5439	size_t ret;
5440	int err;
5441
5442	ret = cnt;
5443
5444	if (cnt > MAX_TRACER_SIZE)
5445		cnt = MAX_TRACER_SIZE;
5446
5447	if (copy_from_user(buf, ubuf, cnt))
5448		return -EFAULT;
5449
5450	buf[cnt] = 0;
5451
5452	/* strip ending whitespace. */
5453	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5454		buf[i] = 0;
5455
5456	err = tracing_set_tracer(tr, buf);
5457	if (err)
5458		return err;
5459
5460	*ppos += ret;
5461
5462	return ret;
5463}
5464
5465static ssize_t
5466tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5467		   size_t cnt, loff_t *ppos)
5468{
5469	char buf[64];
5470	int r;
5471
5472	r = snprintf(buf, sizeof(buf), "%ld\n",
5473		     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5474	if (r > sizeof(buf))
5475		r = sizeof(buf);
5476	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5477}
5478
5479static ssize_t
5480tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5481		    size_t cnt, loff_t *ppos)
5482{
5483	unsigned long val;
5484	int ret;
5485
5486	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5487	if (ret)
5488		return ret;
5489
5490	*ptr = val * 1000;
5491
5492	return cnt;
5493}
5494
5495static ssize_t
5496tracing_thresh_read(struct file *filp, char __user *ubuf,
5497		    size_t cnt, loff_t *ppos)
5498{
5499	return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5500}
5501
5502static ssize_t
5503tracing_thresh_write(struct file *filp, const char __user *ubuf,
5504		     size_t cnt, loff_t *ppos)
5505{
5506	struct trace_array *tr = filp->private_data;
5507	int ret;
5508
5509	mutex_lock(&trace_types_lock);
5510	ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5511	if (ret < 0)
5512		goto out;
5513
5514	if (tr->current_trace->update_thresh) {
5515		ret = tr->current_trace->update_thresh(tr);
5516		if (ret < 0)
5517			goto out;
5518	}
5519
5520	ret = cnt;
5521out:
5522	mutex_unlock(&trace_types_lock);
5523
5524	return ret;
5525}
5526
5527#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5528
5529static ssize_t
5530tracing_max_lat_read(struct file *filp, char __user *ubuf,
5531		     size_t cnt, loff_t *ppos)
5532{
5533	return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5534}
5535
5536static ssize_t
5537tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5538		      size_t cnt, loff_t *ppos)
5539{
5540	return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5541}
5542
5543#endif
5544
5545static int tracing_open_pipe(struct inode *inode, struct file *filp)
5546{
5547	struct trace_array *tr = inode->i_private;
5548	struct trace_iterator *iter;
5549	int ret = 0;
5550
5551	if (tracing_disabled)
5552		return -ENODEV;
5553
5554	if (trace_array_get(tr) < 0)
5555		return -ENODEV;
5556
5557	mutex_lock(&trace_types_lock);
5558
5559	/* create a buffer to store the information to pass to userspace */
5560	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5561	if (!iter) {
5562		ret = -ENOMEM;
5563		__trace_array_put(tr);
5564		goto out;
5565	}
5566
5567	trace_seq_init(&iter->seq);
5568	iter->trace = tr->current_trace;
5569
5570	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5571		ret = -ENOMEM;
5572		goto fail;
5573	}
5574
5575	/* trace pipe does not show start of buffer */
5576	cpumask_setall(iter->started);
5577
5578	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5579		iter->iter_flags |= TRACE_FILE_LAT_FMT;
5580
5581	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
5582	if (trace_clocks[tr->clock_id].in_ns)
5583		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5584
5585	iter->tr = tr;
5586	iter->trace_buffer = &tr->trace_buffer;
5587	iter->cpu_file = tracing_get_cpu(inode);
5588	mutex_init(&iter->mutex);
5589	filp->private_data = iter;
5590
5591	if (iter->trace->pipe_open)
5592		iter->trace->pipe_open(iter);
5593
5594	nonseekable_open(inode, filp);
5595
5596	tr->current_trace->ref++;
5597out:
5598	mutex_unlock(&trace_types_lock);
5599	return ret;
5600
5601fail:
5602	kfree(iter->trace);
5603	kfree(iter);
5604	__trace_array_put(tr);
5605	mutex_unlock(&trace_types_lock);
5606	return ret;
5607}
5608
5609static int tracing_release_pipe(struct inode *inode, struct file *file)
5610{
5611	struct trace_iterator *iter = file->private_data;
5612	struct trace_array *tr = inode->i_private;
5613
5614	mutex_lock(&trace_types_lock);
5615
5616	tr->current_trace->ref--;
5617
5618	if (iter->trace->pipe_close)
5619		iter->trace->pipe_close(iter);
5620
5621	mutex_unlock(&trace_types_lock);
5622
5623	free_cpumask_var(iter->started);
5624	mutex_destroy(&iter->mutex);
5625	kfree(iter);
5626
5627	trace_array_put(tr);
5628
5629	return 0;
5630}
5631
5632static __poll_t
5633trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5634{
5635	struct trace_array *tr = iter->tr;
5636
5637	/* Iterators are static, they should be filled or empty */
5638	if (trace_buffer_iter(iter, iter->cpu_file))
5639		return EPOLLIN | EPOLLRDNORM;
5640
5641	if (tr->trace_flags & TRACE_ITER_BLOCK)
5642		/*
5643		 * Always select as readable when in blocking mode
5644		 */
5645		return EPOLLIN | EPOLLRDNORM;
5646	else
5647		return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
5648					     filp, poll_table);
5649}
5650
5651static __poll_t
5652tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5653{
5654	struct trace_iterator *iter = filp->private_data;
5655
5656	return trace_poll(iter, filp, poll_table);
5657}
5658
5659/* Must be called with iter->mutex held. */
5660static int tracing_wait_pipe(struct file *filp)
5661{
5662	struct trace_iterator *iter = filp->private_data;
5663	int ret;
5664
5665	while (trace_empty(iter)) {
5666
5667		if ((filp->f_flags & O_NONBLOCK)) {
5668			return -EAGAIN;
5669		}
5670
5671		/*
5672		 * We block until we read something and tracing is disabled.
5673		 * We still block if tracing is disabled, but we have never
5674		 * read anything. This allows a user to cat this file, and
5675		 * then enable tracing. But after we have read something,
5676		 * we give an EOF when tracing is again disabled.
5677		 *
5678		 * iter->pos will be 0 if we haven't read anything.
5679		 */
5680		if (!tracer_tracing_is_on(iter->tr) && iter->pos)
5681			break;
5682
5683		mutex_unlock(&iter->mutex);
5684
5685		ret = wait_on_pipe(iter, false);
5686
5687		mutex_lock(&iter->mutex);
5688
5689		if (ret)
5690			return ret;
5691	}
5692
5693	return 1;
5694}
5695
5696/*
5697 * Consumer reader.
5698 */
5699static ssize_t
5700tracing_read_pipe(struct file *filp, char __user *ubuf,
5701		  size_t cnt, loff_t *ppos)
5702{
5703	struct trace_iterator *iter = filp->private_data;
5704	ssize_t sret;
5705
5706	/*
5707	 * Avoid more than one consumer on a single file descriptor
5708	 * This is just a matter of traces coherency, the ring buffer itself
5709	 * is protected.
5710	 */
5711	mutex_lock(&iter->mutex);
5712
5713	/* return any leftover data */
5714	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5715	if (sret != -EBUSY)
5716		goto out;
5717
5718	trace_seq_init(&iter->seq);
5719
 
 
 
 
 
 
5720	if (iter->trace->read) {
5721		sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5722		if (sret)
5723			goto out;
5724	}
5725
5726waitagain:
5727	sret = tracing_wait_pipe(filp);
5728	if (sret <= 0)
5729		goto out;
5730
5731	/* stop when tracing is finished */
5732	if (trace_empty(iter)) {
5733		sret = 0;
5734		goto out;
5735	}
5736
5737	if (cnt >= PAGE_SIZE)
5738		cnt = PAGE_SIZE - 1;
5739
5740	/* reset all but tr, trace, and overruns */
5741	memset(&iter->seq, 0,
5742	       sizeof(struct trace_iterator) -
5743	       offsetof(struct trace_iterator, seq));
5744	cpumask_clear(iter->started);
5745	iter->pos = -1;
5746
5747	trace_event_read_lock();
5748	trace_access_lock(iter->cpu_file);
5749	while (trace_find_next_entry_inc(iter) != NULL) {
5750		enum print_line_t ret;
5751		int save_len = iter->seq.seq.len;
5752
5753		ret = print_trace_line(iter);
5754		if (ret == TRACE_TYPE_PARTIAL_LINE) {
5755			/* don't print partial lines */
5756			iter->seq.seq.len = save_len;
5757			break;
5758		}
5759		if (ret != TRACE_TYPE_NO_CONSUME)
5760			trace_consume(iter);
5761
5762		if (trace_seq_used(&iter->seq) >= cnt)
5763			break;
5764
5765		/*
5766		 * Setting the full flag means we reached the trace_seq buffer
5767		 * size and we should leave by partial output condition above.
5768		 * One of the trace_seq_* functions is not used properly.
5769		 */
5770		WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5771			  iter->ent->type);
5772	}
5773	trace_access_unlock(iter->cpu_file);
5774	trace_event_read_unlock();
5775
5776	/* Now copy what we have to the user */
5777	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5778	if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
5779		trace_seq_init(&iter->seq);
5780
5781	/*
5782	 * If there was nothing to send to user, in spite of consuming trace
5783	 * entries, go back to wait for more entries.
5784	 */
5785	if (sret == -EBUSY)
5786		goto waitagain;
5787
5788out:
5789	mutex_unlock(&iter->mutex);
5790
5791	return sret;
5792}
5793
5794static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5795				     unsigned int idx)
5796{
5797	__free_page(spd->pages[idx]);
5798}
5799
5800static const struct pipe_buf_operations tracing_pipe_buf_ops = {
5801	.can_merge		= 0,
5802	.confirm		= generic_pipe_buf_confirm,
5803	.release		= generic_pipe_buf_release,
5804	.steal			= generic_pipe_buf_steal,
5805	.get			= generic_pipe_buf_get,
5806};
5807
5808static size_t
5809tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5810{
5811	size_t count;
5812	int save_len;
5813	int ret;
5814
5815	/* Seq buffer is page-sized, exactly what we need. */
5816	for (;;) {
5817		save_len = iter->seq.seq.len;
5818		ret = print_trace_line(iter);
5819
5820		if (trace_seq_has_overflowed(&iter->seq)) {
5821			iter->seq.seq.len = save_len;
5822			break;
5823		}
5824
5825		/*
5826		 * This should not be hit, because it should only
5827		 * be set if the iter->seq overflowed. But check it
5828		 * anyway to be safe.
5829		 */
5830		if (ret == TRACE_TYPE_PARTIAL_LINE) {
5831			iter->seq.seq.len = save_len;
5832			break;
5833		}
5834
5835		count = trace_seq_used(&iter->seq) - save_len;
5836		if (rem < count) {
5837			rem = 0;
5838			iter->seq.seq.len = save_len;
5839			break;
5840		}
5841
5842		if (ret != TRACE_TYPE_NO_CONSUME)
5843			trace_consume(iter);
5844		rem -= count;
5845		if (!trace_find_next_entry_inc(iter))	{
5846			rem = 0;
5847			iter->ent = NULL;
5848			break;
5849		}
5850	}
5851
5852	return rem;
5853}
5854
5855static ssize_t tracing_splice_read_pipe(struct file *filp,
5856					loff_t *ppos,
5857					struct pipe_inode_info *pipe,
5858					size_t len,
5859					unsigned int flags)
5860{
5861	struct page *pages_def[PIPE_DEF_BUFFERS];
5862	struct partial_page partial_def[PIPE_DEF_BUFFERS];
5863	struct trace_iterator *iter = filp->private_data;
5864	struct splice_pipe_desc spd = {
5865		.pages		= pages_def,
5866		.partial	= partial_def,
5867		.nr_pages	= 0, /* This gets updated below. */
5868		.nr_pages_max	= PIPE_DEF_BUFFERS,
 
5869		.ops		= &tracing_pipe_buf_ops,
5870		.spd_release	= tracing_spd_release_pipe,
5871	};
5872	ssize_t ret;
5873	size_t rem;
5874	unsigned int i;
5875
5876	if (splice_grow_spd(pipe, &spd))
5877		return -ENOMEM;
5878
5879	mutex_lock(&iter->mutex);
5880
5881	if (iter->trace->splice_read) {
5882		ret = iter->trace->splice_read(iter, filp,
5883					       ppos, pipe, len, flags);
5884		if (ret)
5885			goto out_err;
5886	}
5887
5888	ret = tracing_wait_pipe(filp);
5889	if (ret <= 0)
5890		goto out_err;
5891
5892	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5893		ret = -EFAULT;
5894		goto out_err;
5895	}
5896
5897	trace_event_read_lock();
5898	trace_access_lock(iter->cpu_file);
5899
5900	/* Fill as many pages as possible. */
5901	for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5902		spd.pages[i] = alloc_page(GFP_KERNEL);
5903		if (!spd.pages[i])
5904			break;
5905
5906		rem = tracing_fill_pipe_page(rem, iter);
5907
5908		/* Copy the data into the page, so we can start over. */
5909		ret = trace_seq_to_buffer(&iter->seq,
5910					  page_address(spd.pages[i]),
5911					  trace_seq_used(&iter->seq));
5912		if (ret < 0) {
5913			__free_page(spd.pages[i]);
5914			break;
5915		}
5916		spd.partial[i].offset = 0;
5917		spd.partial[i].len = trace_seq_used(&iter->seq);
5918
5919		trace_seq_init(&iter->seq);
5920	}
5921
5922	trace_access_unlock(iter->cpu_file);
5923	trace_event_read_unlock();
5924	mutex_unlock(&iter->mutex);
5925
5926	spd.nr_pages = i;
5927
5928	if (i)
5929		ret = splice_to_pipe(pipe, &spd);
5930	else
5931		ret = 0;
5932out:
5933	splice_shrink_spd(&spd);
5934	return ret;
5935
5936out_err:
5937	mutex_unlock(&iter->mutex);
5938	goto out;
5939}
5940
5941static ssize_t
5942tracing_entries_read(struct file *filp, char __user *ubuf,
5943		     size_t cnt, loff_t *ppos)
5944{
5945	struct inode *inode = file_inode(filp);
5946	struct trace_array *tr = inode->i_private;
5947	int cpu = tracing_get_cpu(inode);
5948	char buf[64];
5949	int r = 0;
5950	ssize_t ret;
5951
5952	mutex_lock(&trace_types_lock);
5953
5954	if (cpu == RING_BUFFER_ALL_CPUS) {
5955		int cpu, buf_size_same;
5956		unsigned long size;
5957
5958		size = 0;
5959		buf_size_same = 1;
5960		/* check if all cpu sizes are same */
5961		for_each_tracing_cpu(cpu) {
5962			/* fill in the size from first enabled cpu */
5963			if (size == 0)
5964				size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5965			if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
5966				buf_size_same = 0;
5967				break;
5968			}
5969		}
5970
5971		if (buf_size_same) {
5972			if (!ring_buffer_expanded)
5973				r = sprintf(buf, "%lu (expanded: %lu)\n",
5974					    size >> 10,
5975					    trace_buf_size >> 10);
5976			else
5977				r = sprintf(buf, "%lu\n", size >> 10);
5978		} else
5979			r = sprintf(buf, "X\n");
5980	} else
5981		r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5982
5983	mutex_unlock(&trace_types_lock);
5984
5985	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5986	return ret;
5987}
5988
5989static ssize_t
5990tracing_entries_write(struct file *filp, const char __user *ubuf,
5991		      size_t cnt, loff_t *ppos)
5992{
5993	struct inode *inode = file_inode(filp);
5994	struct trace_array *tr = inode->i_private;
5995	unsigned long val;
5996	int ret;
5997
5998	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5999	if (ret)
6000		return ret;
6001
6002	/* must have at least 1 entry */
6003	if (!val)
6004		return -EINVAL;
6005
6006	/* value is in KB */
6007	val <<= 10;
6008	ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6009	if (ret < 0)
6010		return ret;
6011
6012	*ppos += cnt;
6013
6014	return cnt;
6015}
6016
6017static ssize_t
6018tracing_total_entries_read(struct file *filp, char __user *ubuf,
6019				size_t cnt, loff_t *ppos)
6020{
6021	struct trace_array *tr = filp->private_data;
6022	char buf[64];
6023	int r, cpu;
6024	unsigned long size = 0, expanded_size = 0;
6025
6026	mutex_lock(&trace_types_lock);
6027	for_each_tracing_cpu(cpu) {
6028		size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
6029		if (!ring_buffer_expanded)
6030			expanded_size += trace_buf_size >> 10;
6031	}
6032	if (ring_buffer_expanded)
6033		r = sprintf(buf, "%lu\n", size);
6034	else
6035		r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6036	mutex_unlock(&trace_types_lock);
6037
6038	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6039}
6040
6041static ssize_t
6042tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6043			  size_t cnt, loff_t *ppos)
6044{
6045	/*
6046	 * There is no need to read what the user has written, this function
6047	 * is just to make sure that there is no error when "echo" is used
6048	 */
6049
6050	*ppos += cnt;
6051
6052	return cnt;
6053}
6054
6055static int
6056tracing_free_buffer_release(struct inode *inode, struct file *filp)
6057{
6058	struct trace_array *tr = inode->i_private;
6059
6060	/* disable tracing ? */
6061	if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6062		tracer_tracing_off(tr);
6063	/* resize the ring buffer to 0 */
6064	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6065
6066	trace_array_put(tr);
6067
6068	return 0;
6069}
6070
6071static ssize_t
6072tracing_mark_write(struct file *filp, const char __user *ubuf,
6073					size_t cnt, loff_t *fpos)
6074{
 
6075	struct trace_array *tr = filp->private_data;
6076	struct ring_buffer_event *event;
6077	struct ring_buffer *buffer;
6078	struct print_entry *entry;
6079	unsigned long irq_flags;
6080	const char faulted[] = "<faulted>";
 
 
6081	ssize_t written;
 
6082	int size;
6083	int len;
6084
6085/* Used in tracing_mark_raw_write() as well */
6086#define FAULTED_SIZE (sizeof(faulted) - 1) /* '\0' is already accounted for */
6087
6088	if (tracing_disabled)
6089		return -EINVAL;
6090
6091	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6092		return -EINVAL;
6093
6094	if (cnt > TRACE_BUF_SIZE)
6095		cnt = TRACE_BUF_SIZE;
6096
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6097	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6098
6099	local_save_flags(irq_flags);
6100	size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
 
 
 
 
 
 
 
 
 
 
 
 
6101
6102	/* If less than "<faulted>", then make sure we can still add that */
6103	if (cnt < FAULTED_SIZE)
6104		size += FAULTED_SIZE - cnt;
6105
 
 
6106	buffer = tr->trace_buffer.buffer;
6107	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6108					    irq_flags, preempt_count());
6109	if (unlikely(!event))
6110		/* Ring buffer disabled, return as if not open for write */
6111		return -EBADF;
 
 
6112
6113	entry = ring_buffer_event_data(event);
6114	entry->ip = _THIS_IP_;
6115
6116	len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6117	if (len) {
6118		memcpy(&entry->buf, faulted, FAULTED_SIZE);
6119		cnt = FAULTED_SIZE;
6120		written = -EFAULT;
6121	} else
6122		written = cnt;
6123	len = cnt;
6124
6125	if (entry->buf[cnt - 1] != '\n') {
6126		entry->buf[cnt] = '\n';
6127		entry->buf[cnt + 1] = '\0';
6128	} else
6129		entry->buf[cnt] = '\0';
6130
6131	__buffer_unlock_commit(buffer, event);
6132
6133	if (written > 0)
6134		*fpos += written;
6135
6136	return written;
6137}
6138
6139/* Limit it for now to 3K (including tag) */
6140#define RAW_DATA_MAX_SIZE (1024*3)
6141
6142static ssize_t
6143tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6144					size_t cnt, loff_t *fpos)
6145{
6146	struct trace_array *tr = filp->private_data;
6147	struct ring_buffer_event *event;
6148	struct ring_buffer *buffer;
6149	struct raw_data_entry *entry;
6150	const char faulted[] = "<faulted>";
6151	unsigned long irq_flags;
6152	ssize_t written;
6153	int size;
6154	int len;
6155
6156#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6157
6158	if (tracing_disabled)
6159		return -EINVAL;
6160
6161	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6162		return -EINVAL;
6163
6164	/* The marker must at least have a tag id */
6165	if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6166		return -EINVAL;
6167
6168	if (cnt > TRACE_BUF_SIZE)
6169		cnt = TRACE_BUF_SIZE;
6170
6171	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6172
6173	local_save_flags(irq_flags);
6174	size = sizeof(*entry) + cnt;
6175	if (cnt < FAULT_SIZE_ID)
6176		size += FAULT_SIZE_ID - cnt;
6177
6178	buffer = tr->trace_buffer.buffer;
6179	event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6180					    irq_flags, preempt_count());
6181	if (!event)
6182		/* Ring buffer disabled, return as if not open for write */
6183		return -EBADF;
6184
6185	entry = ring_buffer_event_data(event);
6186
6187	len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6188	if (len) {
6189		entry->id = -1;
6190		memcpy(&entry->buf, faulted, FAULTED_SIZE);
6191		written = -EFAULT;
6192	} else
6193		written = cnt;
6194
6195	__buffer_unlock_commit(buffer, event);
6196
6197	if (written > 0)
6198		*fpos += written;
6199
 
 
 
 
 
 
6200	return written;
6201}
6202
6203static int tracing_clock_show(struct seq_file *m, void *v)
6204{
6205	struct trace_array *tr = m->private;
6206	int i;
6207
6208	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6209		seq_printf(m,
6210			"%s%s%s%s", i ? " " : "",
6211			i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6212			i == tr->clock_id ? "]" : "");
6213	seq_putc(m, '\n');
6214
6215	return 0;
6216}
6217
6218int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6219{
6220	int i;
6221
6222	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6223		if (strcmp(trace_clocks[i].name, clockstr) == 0)
6224			break;
6225	}
6226	if (i == ARRAY_SIZE(trace_clocks))
6227		return -EINVAL;
6228
6229	mutex_lock(&trace_types_lock);
6230
6231	tr->clock_id = i;
6232
6233	ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
6234
6235	/*
6236	 * New clock may not be consistent with the previous clock.
6237	 * Reset the buffer so that it doesn't have incomparable timestamps.
6238	 */
6239	tracing_reset_online_cpus(&tr->trace_buffer);
6240
6241#ifdef CONFIG_TRACER_MAX_TRACE
6242	if (tr->max_buffer.buffer)
6243		ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6244	tracing_reset_online_cpus(&tr->max_buffer);
6245#endif
6246
6247	mutex_unlock(&trace_types_lock);
6248
6249	return 0;
6250}
6251
6252static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6253				   size_t cnt, loff_t *fpos)
6254{
6255	struct seq_file *m = filp->private_data;
6256	struct trace_array *tr = m->private;
6257	char buf[64];
6258	const char *clockstr;
6259	int ret;
6260
6261	if (cnt >= sizeof(buf))
6262		return -EINVAL;
6263
6264	if (copy_from_user(buf, ubuf, cnt))
6265		return -EFAULT;
6266
6267	buf[cnt] = 0;
6268
6269	clockstr = strstrip(buf);
6270
6271	ret = tracing_set_clock(tr, clockstr);
6272	if (ret)
6273		return ret;
6274
6275	*fpos += cnt;
6276
6277	return cnt;
6278}
6279
6280static int tracing_clock_open(struct inode *inode, struct file *file)
6281{
6282	struct trace_array *tr = inode->i_private;
6283	int ret;
6284
6285	if (tracing_disabled)
6286		return -ENODEV;
6287
6288	if (trace_array_get(tr))
6289		return -ENODEV;
6290
6291	ret = single_open(file, tracing_clock_show, inode->i_private);
6292	if (ret < 0)
6293		trace_array_put(tr);
6294
6295	return ret;
6296}
6297
6298static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6299{
6300	struct trace_array *tr = m->private;
6301
6302	mutex_lock(&trace_types_lock);
6303
6304	if (ring_buffer_time_stamp_abs(tr->trace_buffer.buffer))
6305		seq_puts(m, "delta [absolute]\n");
6306	else
6307		seq_puts(m, "[delta] absolute\n");
6308
6309	mutex_unlock(&trace_types_lock);
6310
6311	return 0;
6312}
6313
6314static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6315{
6316	struct trace_array *tr = inode->i_private;
6317	int ret;
6318
6319	if (tracing_disabled)
6320		return -ENODEV;
6321
6322	if (trace_array_get(tr))
6323		return -ENODEV;
6324
6325	ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6326	if (ret < 0)
6327		trace_array_put(tr);
6328
6329	return ret;
6330}
6331
6332int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6333{
6334	int ret = 0;
6335
6336	mutex_lock(&trace_types_lock);
6337
6338	if (abs && tr->time_stamp_abs_ref++)
6339		goto out;
6340
6341	if (!abs) {
6342		if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6343			ret = -EINVAL;
6344			goto out;
6345		}
6346
6347		if (--tr->time_stamp_abs_ref)
6348			goto out;
6349	}
6350
6351	ring_buffer_set_time_stamp_abs(tr->trace_buffer.buffer, abs);
6352
6353#ifdef CONFIG_TRACER_MAX_TRACE
6354	if (tr->max_buffer.buffer)
6355		ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
6356#endif
6357 out:
6358	mutex_unlock(&trace_types_lock);
6359
6360	return ret;
6361}
6362
6363struct ftrace_buffer_info {
6364	struct trace_iterator	iter;
6365	void			*spare;
6366	unsigned int		spare_cpu;
6367	unsigned int		read;
6368};
6369
6370#ifdef CONFIG_TRACER_SNAPSHOT
6371static int tracing_snapshot_open(struct inode *inode, struct file *file)
6372{
6373	struct trace_array *tr = inode->i_private;
6374	struct trace_iterator *iter;
6375	struct seq_file *m;
6376	int ret = 0;
6377
6378	if (trace_array_get(tr) < 0)
6379		return -ENODEV;
6380
6381	if (file->f_mode & FMODE_READ) {
6382		iter = __tracing_open(inode, file, true);
6383		if (IS_ERR(iter))
6384			ret = PTR_ERR(iter);
6385	} else {
6386		/* Writes still need the seq_file to hold the private data */
6387		ret = -ENOMEM;
6388		m = kzalloc(sizeof(*m), GFP_KERNEL);
6389		if (!m)
6390			goto out;
6391		iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6392		if (!iter) {
6393			kfree(m);
6394			goto out;
6395		}
6396		ret = 0;
6397
6398		iter->tr = tr;
6399		iter->trace_buffer = &tr->max_buffer;
6400		iter->cpu_file = tracing_get_cpu(inode);
6401		m->private = iter;
6402		file->private_data = m;
6403	}
6404out:
6405	if (ret < 0)
6406		trace_array_put(tr);
6407
6408	return ret;
6409}
6410
6411static ssize_t
6412tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6413		       loff_t *ppos)
6414{
6415	struct seq_file *m = filp->private_data;
6416	struct trace_iterator *iter = m->private;
6417	struct trace_array *tr = iter->tr;
6418	unsigned long val;
6419	int ret;
6420
6421	ret = tracing_update_buffers();
6422	if (ret < 0)
6423		return ret;
6424
6425	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6426	if (ret)
6427		return ret;
6428
6429	mutex_lock(&trace_types_lock);
6430
6431	if (tr->current_trace->use_max_tr) {
6432		ret = -EBUSY;
6433		goto out;
6434	}
6435
6436	switch (val) {
6437	case 0:
6438		if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6439			ret = -EINVAL;
6440			break;
6441		}
6442		if (tr->allocated_snapshot)
6443			free_snapshot(tr);
6444		break;
6445	case 1:
6446/* Only allow per-cpu swap if the ring buffer supports it */
6447#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6448		if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6449			ret = -EINVAL;
6450			break;
6451		}
6452#endif
6453		if (!tr->allocated_snapshot) {
6454			ret = tracing_alloc_snapshot_instance(tr);
6455			if (ret < 0)
6456				break;
6457		}
6458		local_irq_disable();
6459		/* Now, we're going to swap */
6460		if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6461			update_max_tr(tr, current, smp_processor_id());
6462		else
6463			update_max_tr_single(tr, current, iter->cpu_file);
6464		local_irq_enable();
6465		break;
6466	default:
6467		if (tr->allocated_snapshot) {
6468			if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6469				tracing_reset_online_cpus(&tr->max_buffer);
6470			else
6471				tracing_reset(&tr->max_buffer, iter->cpu_file);
6472		}
6473		break;
6474	}
6475
6476	if (ret >= 0) {
6477		*ppos += cnt;
6478		ret = cnt;
6479	}
6480out:
6481	mutex_unlock(&trace_types_lock);
6482	return ret;
6483}
6484
6485static int tracing_snapshot_release(struct inode *inode, struct file *file)
6486{
6487	struct seq_file *m = file->private_data;
6488	int ret;
6489
6490	ret = tracing_release(inode, file);
6491
6492	if (file->f_mode & FMODE_READ)
6493		return ret;
6494
6495	/* If write only, the seq_file is just a stub */
6496	if (m)
6497		kfree(m->private);
6498	kfree(m);
6499
6500	return 0;
6501}
6502
6503static int tracing_buffers_open(struct inode *inode, struct file *filp);
6504static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
6505				    size_t count, loff_t *ppos);
6506static int tracing_buffers_release(struct inode *inode, struct file *file);
6507static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6508		   struct pipe_inode_info *pipe, size_t len, unsigned int flags);
6509
6510static int snapshot_raw_open(struct inode *inode, struct file *filp)
6511{
6512	struct ftrace_buffer_info *info;
6513	int ret;
6514
6515	ret = tracing_buffers_open(inode, filp);
6516	if (ret < 0)
6517		return ret;
6518
6519	info = filp->private_data;
6520
6521	if (info->iter.trace->use_max_tr) {
6522		tracing_buffers_release(inode, filp);
6523		return -EBUSY;
6524	}
6525
6526	info->iter.snapshot = true;
6527	info->iter.trace_buffer = &info->iter.tr->max_buffer;
6528
6529	return ret;
6530}
6531
6532#endif /* CONFIG_TRACER_SNAPSHOT */
6533
6534
6535static const struct file_operations tracing_thresh_fops = {
6536	.open		= tracing_open_generic,
6537	.read		= tracing_thresh_read,
6538	.write		= tracing_thresh_write,
6539	.llseek		= generic_file_llseek,
6540};
6541
6542#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6543static const struct file_operations tracing_max_lat_fops = {
6544	.open		= tracing_open_generic,
6545	.read		= tracing_max_lat_read,
6546	.write		= tracing_max_lat_write,
6547	.llseek		= generic_file_llseek,
6548};
6549#endif
6550
6551static const struct file_operations set_tracer_fops = {
6552	.open		= tracing_open_generic,
6553	.read		= tracing_set_trace_read,
6554	.write		= tracing_set_trace_write,
6555	.llseek		= generic_file_llseek,
6556};
6557
6558static const struct file_operations tracing_pipe_fops = {
6559	.open		= tracing_open_pipe,
6560	.poll		= tracing_poll_pipe,
6561	.read		= tracing_read_pipe,
6562	.splice_read	= tracing_splice_read_pipe,
6563	.release	= tracing_release_pipe,
6564	.llseek		= no_llseek,
6565};
6566
6567static const struct file_operations tracing_entries_fops = {
6568	.open		= tracing_open_generic_tr,
6569	.read		= tracing_entries_read,
6570	.write		= tracing_entries_write,
6571	.llseek		= generic_file_llseek,
6572	.release	= tracing_release_generic_tr,
6573};
6574
6575static const struct file_operations tracing_total_entries_fops = {
6576	.open		= tracing_open_generic_tr,
6577	.read		= tracing_total_entries_read,
6578	.llseek		= generic_file_llseek,
6579	.release	= tracing_release_generic_tr,
6580};
6581
6582static const struct file_operations tracing_free_buffer_fops = {
6583	.open		= tracing_open_generic_tr,
6584	.write		= tracing_free_buffer_write,
6585	.release	= tracing_free_buffer_release,
6586};
6587
6588static const struct file_operations tracing_mark_fops = {
6589	.open		= tracing_open_generic_tr,
6590	.write		= tracing_mark_write,
6591	.llseek		= generic_file_llseek,
6592	.release	= tracing_release_generic_tr,
6593};
6594
6595static const struct file_operations tracing_mark_raw_fops = {
6596	.open		= tracing_open_generic_tr,
6597	.write		= tracing_mark_raw_write,
6598	.llseek		= generic_file_llseek,
6599	.release	= tracing_release_generic_tr,
6600};
6601
6602static const struct file_operations trace_clock_fops = {
6603	.open		= tracing_clock_open,
6604	.read		= seq_read,
6605	.llseek		= seq_lseek,
6606	.release	= tracing_single_release_tr,
6607	.write		= tracing_clock_write,
6608};
6609
6610static const struct file_operations trace_time_stamp_mode_fops = {
6611	.open		= tracing_time_stamp_mode_open,
6612	.read		= seq_read,
6613	.llseek		= seq_lseek,
6614	.release	= tracing_single_release_tr,
6615};
6616
6617#ifdef CONFIG_TRACER_SNAPSHOT
6618static const struct file_operations snapshot_fops = {
6619	.open		= tracing_snapshot_open,
6620	.read		= seq_read,
6621	.write		= tracing_snapshot_write,
6622	.llseek		= tracing_lseek,
6623	.release	= tracing_snapshot_release,
6624};
6625
6626static const struct file_operations snapshot_raw_fops = {
6627	.open		= snapshot_raw_open,
6628	.read		= tracing_buffers_read,
6629	.release	= tracing_buffers_release,
6630	.splice_read	= tracing_buffers_splice_read,
6631	.llseek		= no_llseek,
6632};
6633
6634#endif /* CONFIG_TRACER_SNAPSHOT */
6635
6636static int tracing_buffers_open(struct inode *inode, struct file *filp)
6637{
6638	struct trace_array *tr = inode->i_private;
6639	struct ftrace_buffer_info *info;
6640	int ret;
6641
6642	if (tracing_disabled)
6643		return -ENODEV;
6644
6645	if (trace_array_get(tr) < 0)
6646		return -ENODEV;
6647
6648	info = kzalloc(sizeof(*info), GFP_KERNEL);
6649	if (!info) {
6650		trace_array_put(tr);
6651		return -ENOMEM;
6652	}
6653
6654	mutex_lock(&trace_types_lock);
6655
6656	info->iter.tr		= tr;
6657	info->iter.cpu_file	= tracing_get_cpu(inode);
6658	info->iter.trace	= tr->current_trace;
6659	info->iter.trace_buffer = &tr->trace_buffer;
6660	info->spare		= NULL;
6661	/* Force reading ring buffer for first read */
6662	info->read		= (unsigned int)-1;
6663
6664	filp->private_data = info;
6665
6666	tr->current_trace->ref++;
6667
6668	mutex_unlock(&trace_types_lock);
6669
6670	ret = nonseekable_open(inode, filp);
6671	if (ret < 0)
6672		trace_array_put(tr);
6673
6674	return ret;
6675}
6676
6677static __poll_t
6678tracing_buffers_poll(struct file *filp, poll_table *poll_table)
6679{
6680	struct ftrace_buffer_info *info = filp->private_data;
6681	struct trace_iterator *iter = &info->iter;
6682
6683	return trace_poll(iter, filp, poll_table);
6684}
6685
6686static ssize_t
6687tracing_buffers_read(struct file *filp, char __user *ubuf,
6688		     size_t count, loff_t *ppos)
6689{
6690	struct ftrace_buffer_info *info = filp->private_data;
6691	struct trace_iterator *iter = &info->iter;
6692	ssize_t ret = 0;
6693	ssize_t size;
6694
6695	if (!count)
6696		return 0;
6697
6698#ifdef CONFIG_TRACER_MAX_TRACE
6699	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6700		return -EBUSY;
6701#endif
6702
6703	if (!info->spare) {
6704		info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
6705							  iter->cpu_file);
6706		if (IS_ERR(info->spare)) {
6707			ret = PTR_ERR(info->spare);
6708			info->spare = NULL;
6709		} else {
6710			info->spare_cpu = iter->cpu_file;
6711		}
6712	}
6713	if (!info->spare)
6714		return ret;
6715
6716	/* Do we have previous read data to read? */
6717	if (info->read < PAGE_SIZE)
6718		goto read;
6719
6720 again:
6721	trace_access_lock(iter->cpu_file);
6722	ret = ring_buffer_read_page(iter->trace_buffer->buffer,
6723				    &info->spare,
6724				    count,
6725				    iter->cpu_file, 0);
6726	trace_access_unlock(iter->cpu_file);
6727
6728	if (ret < 0) {
6729		if (trace_empty(iter)) {
6730			if ((filp->f_flags & O_NONBLOCK))
6731				return -EAGAIN;
6732
6733			ret = wait_on_pipe(iter, false);
6734			if (ret)
6735				return ret;
6736
6737			goto again;
6738		}
6739		return 0;
6740	}
6741
6742	info->read = 0;
6743 read:
6744	size = PAGE_SIZE - info->read;
6745	if (size > count)
6746		size = count;
6747
6748	ret = copy_to_user(ubuf, info->spare + info->read, size);
6749	if (ret == size)
6750		return -EFAULT;
6751
6752	size -= ret;
6753
6754	*ppos += size;
6755	info->read += size;
6756
6757	return size;
6758}
6759
6760static int tracing_buffers_release(struct inode *inode, struct file *file)
6761{
6762	struct ftrace_buffer_info *info = file->private_data;
6763	struct trace_iterator *iter = &info->iter;
6764
6765	mutex_lock(&trace_types_lock);
6766
6767	iter->tr->current_trace->ref--;
6768
6769	__trace_array_put(iter->tr);
6770
6771	if (info->spare)
6772		ring_buffer_free_read_page(iter->trace_buffer->buffer,
6773					   info->spare_cpu, info->spare);
6774	kfree(info);
6775
6776	mutex_unlock(&trace_types_lock);
6777
6778	return 0;
6779}
6780
6781struct buffer_ref {
6782	struct ring_buffer	*buffer;
6783	void			*page;
6784	int			cpu;
6785	int			ref;
6786};
6787
6788static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
6789				    struct pipe_buffer *buf)
6790{
6791	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6792
6793	if (--ref->ref)
6794		return;
6795
6796	ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
6797	kfree(ref);
6798	buf->private = 0;
6799}
6800
6801static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
6802				struct pipe_buffer *buf)
6803{
6804	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6805
6806	ref->ref++;
6807}
6808
6809/* Pipe buffer operations for a buffer. */
6810static const struct pipe_buf_operations buffer_pipe_buf_ops = {
6811	.can_merge		= 0,
6812	.confirm		= generic_pipe_buf_confirm,
6813	.release		= buffer_pipe_buf_release,
6814	.steal			= generic_pipe_buf_steal,
6815	.get			= buffer_pipe_buf_get,
6816};
6817
6818/*
6819 * Callback from splice_to_pipe(), if we need to release some pages
6820 * at the end of the spd in case we error'ed out in filling the pipe.
6821 */
6822static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
6823{
6824	struct buffer_ref *ref =
6825		(struct buffer_ref *)spd->partial[i].private;
6826
6827	if (--ref->ref)
6828		return;
6829
6830	ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
6831	kfree(ref);
6832	spd->partial[i].private = 0;
6833}
6834
6835static ssize_t
6836tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6837			    struct pipe_inode_info *pipe, size_t len,
6838			    unsigned int flags)
6839{
6840	struct ftrace_buffer_info *info = file->private_data;
6841	struct trace_iterator *iter = &info->iter;
6842	struct partial_page partial_def[PIPE_DEF_BUFFERS];
6843	struct page *pages_def[PIPE_DEF_BUFFERS];
6844	struct splice_pipe_desc spd = {
6845		.pages		= pages_def,
6846		.partial	= partial_def,
6847		.nr_pages_max	= PIPE_DEF_BUFFERS,
 
6848		.ops		= &buffer_pipe_buf_ops,
6849		.spd_release	= buffer_spd_release,
6850	};
6851	struct buffer_ref *ref;
6852	int entries, i;
6853	ssize_t ret = 0;
6854
6855#ifdef CONFIG_TRACER_MAX_TRACE
6856	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6857		return -EBUSY;
6858#endif
6859
 
 
 
6860	if (*ppos & (PAGE_SIZE - 1))
6861		return -EINVAL;
6862
6863	if (len & (PAGE_SIZE - 1)) {
6864		if (len < PAGE_SIZE)
6865			return -EINVAL;
6866		len &= PAGE_MASK;
6867	}
6868
6869	if (splice_grow_spd(pipe, &spd))
6870		return -ENOMEM;
6871
6872 again:
6873	trace_access_lock(iter->cpu_file);
6874	entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6875
6876	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
6877		struct page *page;
6878		int r;
6879
6880		ref = kzalloc(sizeof(*ref), GFP_KERNEL);
6881		if (!ref) {
6882			ret = -ENOMEM;
6883			break;
6884		}
6885
6886		ref->ref = 1;
6887		ref->buffer = iter->trace_buffer->buffer;
6888		ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
6889		if (IS_ERR(ref->page)) {
6890			ret = PTR_ERR(ref->page);
6891			ref->page = NULL;
6892			kfree(ref);
6893			break;
6894		}
6895		ref->cpu = iter->cpu_file;
6896
6897		r = ring_buffer_read_page(ref->buffer, &ref->page,
6898					  len, iter->cpu_file, 1);
6899		if (r < 0) {
6900			ring_buffer_free_read_page(ref->buffer, ref->cpu,
6901						   ref->page);
6902			kfree(ref);
6903			break;
6904		}
6905
 
 
 
 
 
 
 
 
6906		page = virt_to_page(ref->page);
6907
6908		spd.pages[i] = page;
6909		spd.partial[i].len = PAGE_SIZE;
6910		spd.partial[i].offset = 0;
6911		spd.partial[i].private = (unsigned long)ref;
6912		spd.nr_pages++;
6913		*ppos += PAGE_SIZE;
6914
6915		entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6916	}
6917
6918	trace_access_unlock(iter->cpu_file);
6919	spd.nr_pages = i;
6920
6921	/* did we read anything? */
6922	if (!spd.nr_pages) {
6923		if (ret)
6924			goto out;
6925
6926		ret = -EAGAIN;
6927		if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
6928			goto out;
6929
6930		ret = wait_on_pipe(iter, true);
6931		if (ret)
6932			goto out;
6933
6934		goto again;
6935	}
6936
6937	ret = splice_to_pipe(pipe, &spd);
6938out:
6939	splice_shrink_spd(&spd);
6940
6941	return ret;
6942}
6943
6944static const struct file_operations tracing_buffers_fops = {
6945	.open		= tracing_buffers_open,
6946	.read		= tracing_buffers_read,
6947	.poll		= tracing_buffers_poll,
6948	.release	= tracing_buffers_release,
6949	.splice_read	= tracing_buffers_splice_read,
6950	.llseek		= no_llseek,
6951};
6952
6953static ssize_t
6954tracing_stats_read(struct file *filp, char __user *ubuf,
6955		   size_t count, loff_t *ppos)
6956{
6957	struct inode *inode = file_inode(filp);
6958	struct trace_array *tr = inode->i_private;
6959	struct trace_buffer *trace_buf = &tr->trace_buffer;
6960	int cpu = tracing_get_cpu(inode);
6961	struct trace_seq *s;
6962	unsigned long cnt;
6963	unsigned long long t;
6964	unsigned long usec_rem;
6965
6966	s = kmalloc(sizeof(*s), GFP_KERNEL);
6967	if (!s)
6968		return -ENOMEM;
6969
6970	trace_seq_init(s);
6971
6972	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
6973	trace_seq_printf(s, "entries: %ld\n", cnt);
6974
6975	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
6976	trace_seq_printf(s, "overrun: %ld\n", cnt);
6977
6978	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
6979	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6980
6981	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
6982	trace_seq_printf(s, "bytes: %ld\n", cnt);
6983
6984	if (trace_clocks[tr->clock_id].in_ns) {
6985		/* local or global for trace_clock */
6986		t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6987		usec_rem = do_div(t, USEC_PER_SEC);
6988		trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6989								t, usec_rem);
6990
6991		t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
6992		usec_rem = do_div(t, USEC_PER_SEC);
6993		trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6994	} else {
6995		/* counter or tsc mode for trace_clock */
6996		trace_seq_printf(s, "oldest event ts: %llu\n",
6997				ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6998
6999		trace_seq_printf(s, "now ts: %llu\n",
7000				ring_buffer_time_stamp(trace_buf->buffer, cpu));
7001	}
7002
7003	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7004	trace_seq_printf(s, "dropped events: %ld\n", cnt);
7005
7006	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7007	trace_seq_printf(s, "read events: %ld\n", cnt);
7008
7009	count = simple_read_from_buffer(ubuf, count, ppos,
7010					s->buffer, trace_seq_used(s));
7011
7012	kfree(s);
7013
7014	return count;
7015}
7016
7017static const struct file_operations tracing_stats_fops = {
7018	.open		= tracing_open_generic_tr,
7019	.read		= tracing_stats_read,
7020	.llseek		= generic_file_llseek,
7021	.release	= tracing_release_generic_tr,
7022};
7023
7024#ifdef CONFIG_DYNAMIC_FTRACE
7025
 
 
 
 
 
7026static ssize_t
7027tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7028		  size_t cnt, loff_t *ppos)
7029{
 
 
7030	unsigned long *p = filp->private_data;
7031	char buf[64]; /* Not too big for a shallow stack */
 
7032	int r;
7033
7034	r = scnprintf(buf, 63, "%ld", *p);
 
 
 
7035	buf[r++] = '\n';
7036
7037	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 
 
 
 
7038}
7039
7040static const struct file_operations tracing_dyn_info_fops = {
7041	.open		= tracing_open_generic,
7042	.read		= tracing_read_dyn_info,
7043	.llseek		= generic_file_llseek,
7044};
7045#endif /* CONFIG_DYNAMIC_FTRACE */
7046
7047#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7048static void
7049ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7050		struct trace_array *tr, struct ftrace_probe_ops *ops,
7051		void *data)
7052{
7053	tracing_snapshot_instance(tr);
7054}
7055
7056static void
7057ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7058		      struct trace_array *tr, struct ftrace_probe_ops *ops,
7059		      void *data)
7060{
7061	struct ftrace_func_mapper *mapper = data;
7062	long *count = NULL;
7063
7064	if (mapper)
7065		count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7066
7067	if (count) {
7068
7069		if (*count <= 0)
7070			return;
7071
 
7072		(*count)--;
7073	}
7074
7075	tracing_snapshot_instance(tr);
7076}
7077
7078static int
7079ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7080		      struct ftrace_probe_ops *ops, void *data)
7081{
7082	struct ftrace_func_mapper *mapper = data;
7083	long *count = NULL;
7084
7085	seq_printf(m, "%ps:", (void *)ip);
7086
7087	seq_puts(m, "snapshot");
7088
7089	if (mapper)
7090		count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7091
7092	if (count)
7093		seq_printf(m, ":count=%ld\n", *count);
7094	else
7095		seq_puts(m, ":unlimited\n");
 
 
7096
7097	return 0;
7098}
7099
7100static int
7101ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
7102		     unsigned long ip, void *init_data, void **data)
7103{
7104	struct ftrace_func_mapper *mapper = *data;
7105
7106	if (!mapper) {
7107		mapper = allocate_ftrace_func_mapper();
7108		if (!mapper)
7109			return -ENOMEM;
7110		*data = mapper;
7111	}
7112
7113	return ftrace_func_mapper_add_ip(mapper, ip, init_data);
7114}
7115
7116static void
7117ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
7118		     unsigned long ip, void *data)
7119{
7120	struct ftrace_func_mapper *mapper = data;
7121
7122	if (!ip) {
7123		if (!mapper)
7124			return;
7125		free_ftrace_func_mapper(mapper, NULL);
7126		return;
7127	}
7128
7129	ftrace_func_mapper_remove_ip(mapper, ip);
7130}
7131
7132static struct ftrace_probe_ops snapshot_probe_ops = {
7133	.func			= ftrace_snapshot,
7134	.print			= ftrace_snapshot_print,
7135};
7136
7137static struct ftrace_probe_ops snapshot_count_probe_ops = {
7138	.func			= ftrace_count_snapshot,
7139	.print			= ftrace_snapshot_print,
7140	.init			= ftrace_snapshot_init,
7141	.free			= ftrace_snapshot_free,
7142};
7143
7144static int
7145ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
7146			       char *glob, char *cmd, char *param, int enable)
7147{
7148	struct ftrace_probe_ops *ops;
7149	void *count = (void *)-1;
7150	char *number;
7151	int ret;
7152
7153	if (!tr)
7154		return -ENODEV;
7155
7156	/* hash funcs only work with set_ftrace_filter */
7157	if (!enable)
7158		return -EINVAL;
7159
7160	ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;
7161
7162	if (glob[0] == '!')
7163		return unregister_ftrace_function_probe_func(glob+1, tr, ops);
 
 
7164
7165	if (!param)
7166		goto out_reg;
7167
7168	number = strsep(&param, ":");
7169
7170	if (!strlen(number))
7171		goto out_reg;
7172
7173	/*
7174	 * We use the callback data field (which is a pointer)
7175	 * as our counter.
7176	 */
7177	ret = kstrtoul(number, 0, (unsigned long *)&count);
7178	if (ret)
7179		return ret;
7180
7181 out_reg:
7182	ret = tracing_alloc_snapshot_instance(tr);
7183	if (ret < 0)
7184		goto out;
7185
7186	ret = register_ftrace_function_probe(glob, tr, ops, count);
 
7187
7188 out:
7189	return ret < 0 ? ret : 0;
7190}
7191
7192static struct ftrace_func_command ftrace_snapshot_cmd = {
7193	.name			= "snapshot",
7194	.func			= ftrace_trace_snapshot_callback,
7195};
7196
7197static __init int register_snapshot_cmd(void)
7198{
7199	return register_ftrace_command(&ftrace_snapshot_cmd);
7200}
7201#else
7202static inline __init int register_snapshot_cmd(void) { return 0; }
7203#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7204
7205static struct dentry *tracing_get_dentry(struct trace_array *tr)
7206{
7207	if (WARN_ON(!tr->dir))
7208		return ERR_PTR(-ENODEV);
7209
7210	/* Top directory uses NULL as the parent */
7211	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7212		return NULL;
7213
7214	/* All sub buffers have a descriptor */
7215	return tr->dir;
7216}
7217
7218static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7219{
7220	struct dentry *d_tracer;
7221
7222	if (tr->percpu_dir)
7223		return tr->percpu_dir;
7224
7225	d_tracer = tracing_get_dentry(tr);
7226	if (IS_ERR(d_tracer))
7227		return NULL;
7228
7229	tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
7230
7231	WARN_ONCE(!tr->percpu_dir,
7232		  "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
7233
7234	return tr->percpu_dir;
7235}
7236
7237static struct dentry *
7238trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7239		      void *data, long cpu, const struct file_operations *fops)
7240{
7241	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7242
7243	if (ret) /* See tracing_get_cpu() */
7244		d_inode(ret)->i_cdev = (void *)(cpu + 1);
7245	return ret;
7246}
7247
7248static void
7249tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
7250{
7251	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
7252	struct dentry *d_cpu;
7253	char cpu_dir[30]; /* 30 characters should be more than enough */
7254
7255	if (!d_percpu)
7256		return;
7257
7258	snprintf(cpu_dir, 30, "cpu%ld", cpu);
7259	d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
7260	if (!d_cpu) {
7261		pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
7262		return;
7263	}
7264
7265	/* per cpu trace_pipe */
7266	trace_create_cpu_file("trace_pipe", 0444, d_cpu,
7267				tr, cpu, &tracing_pipe_fops);
7268
7269	/* per cpu trace */
7270	trace_create_cpu_file("trace", 0644, d_cpu,
7271				tr, cpu, &tracing_fops);
7272
7273	trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
7274				tr, cpu, &tracing_buffers_fops);
7275
7276	trace_create_cpu_file("stats", 0444, d_cpu,
7277				tr, cpu, &tracing_stats_fops);
7278
7279	trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
7280				tr, cpu, &tracing_entries_fops);
7281
7282#ifdef CONFIG_TRACER_SNAPSHOT
7283	trace_create_cpu_file("snapshot", 0644, d_cpu,
7284				tr, cpu, &snapshot_fops);
7285
7286	trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
7287				tr, cpu, &snapshot_raw_fops);
7288#endif
7289}
7290
7291#ifdef CONFIG_FTRACE_SELFTEST
7292/* Let selftest have access to static functions in this file */
7293#include "trace_selftest.c"
7294#endif
7295
7296static ssize_t
7297trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7298			loff_t *ppos)
7299{
7300	struct trace_option_dentry *topt = filp->private_data;
7301	char *buf;
7302
7303	if (topt->flags->val & topt->opt->bit)
7304		buf = "1\n";
7305	else
7306		buf = "0\n";
7307
7308	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7309}
7310
7311static ssize_t
7312trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7313			 loff_t *ppos)
7314{
7315	struct trace_option_dentry *topt = filp->private_data;
7316	unsigned long val;
7317	int ret;
7318
7319	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7320	if (ret)
7321		return ret;
7322
7323	if (val != 0 && val != 1)
7324		return -EINVAL;
7325
7326	if (!!(topt->flags->val & topt->opt->bit) != val) {
7327		mutex_lock(&trace_types_lock);
7328		ret = __set_tracer_option(topt->tr, topt->flags,
7329					  topt->opt, !val);
7330		mutex_unlock(&trace_types_lock);
7331		if (ret)
7332			return ret;
7333	}
7334
7335	*ppos += cnt;
7336
7337	return cnt;
7338}
7339
7340
7341static const struct file_operations trace_options_fops = {
7342	.open = tracing_open_generic,
7343	.read = trace_options_read,
7344	.write = trace_options_write,
7345	.llseek	= generic_file_llseek,
7346};
7347
7348/*
7349 * In order to pass in both the trace_array descriptor as well as the index
7350 * to the flag that the trace option file represents, the trace_array
7351 * has a character array of trace_flags_index[], which holds the index
7352 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7353 * The address of this character array is passed to the flag option file
7354 * read/write callbacks.
7355 *
7356 * In order to extract both the index and the trace_array descriptor,
7357 * get_tr_index() uses the following algorithm.
7358 *
7359 *   idx = *ptr;
7360 *
7361 * As the pointer itself contains the address of the index (remember
7362 * index[1] == 1).
7363 *
7364 * Then to get the trace_array descriptor, by subtracting that index
7365 * from the ptr, we get to the start of the index itself.
7366 *
7367 *   ptr - idx == &index[0]
7368 *
7369 * Then a simple container_of() from that pointer gets us to the
7370 * trace_array descriptor.
7371 */
7372static void get_tr_index(void *data, struct trace_array **ptr,
7373			 unsigned int *pindex)
7374{
7375	*pindex = *(unsigned char *)data;
7376
7377	*ptr = container_of(data - *pindex, struct trace_array,
7378			    trace_flags_index);
7379}
7380
7381static ssize_t
7382trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7383			loff_t *ppos)
7384{
7385	void *tr_index = filp->private_data;
7386	struct trace_array *tr;
7387	unsigned int index;
7388	char *buf;
7389
7390	get_tr_index(tr_index, &tr, &index);
7391
7392	if (tr->trace_flags & (1 << index))
7393		buf = "1\n";
7394	else
7395		buf = "0\n";
7396
7397	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7398}
7399
7400static ssize_t
7401trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7402			 loff_t *ppos)
7403{
7404	void *tr_index = filp->private_data;
7405	struct trace_array *tr;
7406	unsigned int index;
7407	unsigned long val;
7408	int ret;
7409
7410	get_tr_index(tr_index, &tr, &index);
7411
7412	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7413	if (ret)
7414		return ret;
7415
7416	if (val != 0 && val != 1)
7417		return -EINVAL;
7418
7419	mutex_lock(&trace_types_lock);
7420	ret = set_tracer_flag(tr, 1 << index, val);
7421	mutex_unlock(&trace_types_lock);
7422
7423	if (ret < 0)
7424		return ret;
7425
7426	*ppos += cnt;
7427
7428	return cnt;
7429}
7430
7431static const struct file_operations trace_options_core_fops = {
7432	.open = tracing_open_generic,
7433	.read = trace_options_core_read,
7434	.write = trace_options_core_write,
7435	.llseek = generic_file_llseek,
7436};
7437
7438struct dentry *trace_create_file(const char *name,
7439				 umode_t mode,
7440				 struct dentry *parent,
7441				 void *data,
7442				 const struct file_operations *fops)
7443{
7444	struct dentry *ret;
7445
7446	ret = tracefs_create_file(name, mode, parent, data, fops);
7447	if (!ret)
7448		pr_warn("Could not create tracefs '%s' entry\n", name);
7449
7450	return ret;
7451}
7452
7453
7454static struct dentry *trace_options_init_dentry(struct trace_array *tr)
7455{
7456	struct dentry *d_tracer;
7457
7458	if (tr->options)
7459		return tr->options;
7460
7461	d_tracer = tracing_get_dentry(tr);
7462	if (IS_ERR(d_tracer))
7463		return NULL;
7464
7465	tr->options = tracefs_create_dir("options", d_tracer);
7466	if (!tr->options) {
7467		pr_warn("Could not create tracefs directory 'options'\n");
7468		return NULL;
7469	}
7470
7471	return tr->options;
7472}
7473
7474static void
7475create_trace_option_file(struct trace_array *tr,
7476			 struct trace_option_dentry *topt,
7477			 struct tracer_flags *flags,
7478			 struct tracer_opt *opt)
7479{
7480	struct dentry *t_options;
7481
7482	t_options = trace_options_init_dentry(tr);
7483	if (!t_options)
7484		return;
7485
7486	topt->flags = flags;
7487	topt->opt = opt;
7488	topt->tr = tr;
7489
7490	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
7491				    &trace_options_fops);
7492
7493}
7494
7495static void
7496create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
7497{
7498	struct trace_option_dentry *topts;
7499	struct trace_options *tr_topts;
7500	struct tracer_flags *flags;
7501	struct tracer_opt *opts;
7502	int cnt;
7503	int i;
7504
7505	if (!tracer)
7506		return;
7507
7508	flags = tracer->flags;
7509
7510	if (!flags || !flags->opts)
7511		return;
7512
7513	/*
7514	 * If this is an instance, only create flags for tracers
7515	 * the instance may have.
7516	 */
7517	if (!trace_ok_for_array(tracer, tr))
7518		return;
7519
7520	for (i = 0; i < tr->nr_topts; i++) {
7521		/* Make sure there's no duplicate flags. */
7522		if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
7523			return;
7524	}
7525
7526	opts = flags->opts;
7527
7528	for (cnt = 0; opts[cnt].name; cnt++)
7529		;
7530
7531	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
7532	if (!topts)
7533		return;
7534
7535	tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
7536			    GFP_KERNEL);
7537	if (!tr_topts) {
7538		kfree(topts);
7539		return;
7540	}
7541
7542	tr->topts = tr_topts;
7543	tr->topts[tr->nr_topts].tracer = tracer;
7544	tr->topts[tr->nr_topts].topts = topts;
7545	tr->nr_topts++;
7546
7547	for (cnt = 0; opts[cnt].name; cnt++) {
7548		create_trace_option_file(tr, &topts[cnt], flags,
7549					 &opts[cnt]);
7550		WARN_ONCE(topts[cnt].entry == NULL,
7551			  "Failed to create trace option: %s",
7552			  opts[cnt].name);
7553	}
7554}
7555
7556static struct dentry *
7557create_trace_option_core_file(struct trace_array *tr,
7558			      const char *option, long index)
7559{
7560	struct dentry *t_options;
7561
7562	t_options = trace_options_init_dentry(tr);
7563	if (!t_options)
7564		return NULL;
7565
7566	return trace_create_file(option, 0644, t_options,
7567				 (void *)&tr->trace_flags_index[index],
7568				 &trace_options_core_fops);
7569}
7570
7571static void create_trace_options_dir(struct trace_array *tr)
7572{
7573	struct dentry *t_options;
7574	bool top_level = tr == &global_trace;
7575	int i;
7576
7577	t_options = trace_options_init_dentry(tr);
7578	if (!t_options)
7579		return;
7580
7581	for (i = 0; trace_options[i]; i++) {
7582		if (top_level ||
7583		    !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
7584			create_trace_option_core_file(tr, trace_options[i], i);
7585	}
7586}
7587
7588static ssize_t
7589rb_simple_read(struct file *filp, char __user *ubuf,
7590	       size_t cnt, loff_t *ppos)
7591{
7592	struct trace_array *tr = filp->private_data;
7593	char buf[64];
7594	int r;
7595
7596	r = tracer_tracing_is_on(tr);
7597	r = sprintf(buf, "%d\n", r);
7598
7599	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7600}
7601
7602static ssize_t
7603rb_simple_write(struct file *filp, const char __user *ubuf,
7604		size_t cnt, loff_t *ppos)
7605{
7606	struct trace_array *tr = filp->private_data;
7607	struct ring_buffer *buffer = tr->trace_buffer.buffer;
7608	unsigned long val;
7609	int ret;
7610
7611	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7612	if (ret)
7613		return ret;
7614
7615	if (buffer) {
7616		mutex_lock(&trace_types_lock);
7617		if (val) {
7618			tracer_tracing_on(tr);
7619			if (tr->current_trace->start)
7620				tr->current_trace->start(tr);
7621		} else {
7622			tracer_tracing_off(tr);
7623			if (tr->current_trace->stop)
7624				tr->current_trace->stop(tr);
7625		}
7626		mutex_unlock(&trace_types_lock);
7627	}
7628
7629	(*ppos)++;
7630
7631	return cnt;
7632}
7633
7634static const struct file_operations rb_simple_fops = {
7635	.open		= tracing_open_generic_tr,
7636	.read		= rb_simple_read,
7637	.write		= rb_simple_write,
7638	.release	= tracing_release_generic_tr,
7639	.llseek		= default_llseek,
7640};
7641
7642struct dentry *trace_instance_dir;
7643
7644static void
7645init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
7646
7647static int
7648allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
7649{
7650	enum ring_buffer_flags rb_flags;
7651
7652	rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
7653
7654	buf->tr = tr;
7655
7656	buf->buffer = ring_buffer_alloc(size, rb_flags);
7657	if (!buf->buffer)
7658		return -ENOMEM;
7659
7660	buf->data = alloc_percpu(struct trace_array_cpu);
7661	if (!buf->data) {
7662		ring_buffer_free(buf->buffer);
7663		buf->buffer = NULL;
7664		return -ENOMEM;
7665	}
7666
7667	/* Allocate the first page for all buffers */
7668	set_buffer_entries(&tr->trace_buffer,
7669			   ring_buffer_size(tr->trace_buffer.buffer, 0));
7670
7671	return 0;
7672}
7673
7674static int allocate_trace_buffers(struct trace_array *tr, int size)
7675{
7676	int ret;
7677
7678	ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
7679	if (ret)
7680		return ret;
7681
7682#ifdef CONFIG_TRACER_MAX_TRACE
7683	ret = allocate_trace_buffer(tr, &tr->max_buffer,
7684				    allocate_snapshot ? size : 1);
7685	if (WARN_ON(ret)) {
7686		ring_buffer_free(tr->trace_buffer.buffer);
7687		tr->trace_buffer.buffer = NULL;
7688		free_percpu(tr->trace_buffer.data);
7689		tr->trace_buffer.data = NULL;
7690		return -ENOMEM;
7691	}
7692	tr->allocated_snapshot = allocate_snapshot;
7693
7694	/*
7695	 * Only the top level trace array gets its snapshot allocated
7696	 * from the kernel command line.
7697	 */
7698	allocate_snapshot = false;
7699#endif
7700	return 0;
7701}
7702
7703static void free_trace_buffer(struct trace_buffer *buf)
7704{
7705	if (buf->buffer) {
7706		ring_buffer_free(buf->buffer);
7707		buf->buffer = NULL;
7708		free_percpu(buf->data);
7709		buf->data = NULL;
7710	}
7711}
7712
7713static void free_trace_buffers(struct trace_array *tr)
7714{
7715	if (!tr)
7716		return;
7717
7718	free_trace_buffer(&tr->trace_buffer);
7719
7720#ifdef CONFIG_TRACER_MAX_TRACE
7721	free_trace_buffer(&tr->max_buffer);
7722#endif
7723}
7724
7725static void init_trace_flags_index(struct trace_array *tr)
7726{
7727	int i;
7728
7729	/* Used by the trace options files */
7730	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
7731		tr->trace_flags_index[i] = i;
7732}
7733
7734static void __update_tracer_options(struct trace_array *tr)
7735{
7736	struct tracer *t;
7737
7738	for (t = trace_types; t; t = t->next)
7739		add_tracer_options(tr, t);
7740}
7741
7742static void update_tracer_options(struct trace_array *tr)
7743{
7744	mutex_lock(&trace_types_lock);
7745	__update_tracer_options(tr);
7746	mutex_unlock(&trace_types_lock);
7747}
7748
7749static int instance_mkdir(const char *name)
7750{
7751	struct trace_array *tr;
7752	int ret;
7753
7754	mutex_lock(&event_mutex);
7755	mutex_lock(&trace_types_lock);
7756
7757	ret = -EEXIST;
7758	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7759		if (tr->name && strcmp(tr->name, name) == 0)
7760			goto out_unlock;
7761	}
7762
7763	ret = -ENOMEM;
7764	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
7765	if (!tr)
7766		goto out_unlock;
7767
7768	tr->name = kstrdup(name, GFP_KERNEL);
7769	if (!tr->name)
7770		goto out_free_tr;
7771
7772	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
7773		goto out_free_tr;
7774
7775	tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
7776
7777	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
7778
7779	raw_spin_lock_init(&tr->start_lock);
7780
7781	tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7782
7783	tr->current_trace = &nop_trace;
7784
7785	INIT_LIST_HEAD(&tr->systems);
7786	INIT_LIST_HEAD(&tr->events);
7787	INIT_LIST_HEAD(&tr->hist_vars);
7788
7789	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
7790		goto out_free_tr;
7791
7792	tr->dir = tracefs_create_dir(name, trace_instance_dir);
7793	if (!tr->dir)
7794		goto out_free_tr;
7795
7796	ret = event_trace_add_tracer(tr->dir, tr);
7797	if (ret) {
7798		tracefs_remove_recursive(tr->dir);
7799		goto out_free_tr;
7800	}
7801
7802	ftrace_init_trace_array(tr);
7803
7804	init_tracer_tracefs(tr, tr->dir);
7805	init_trace_flags_index(tr);
7806	__update_tracer_options(tr);
7807
7808	list_add(&tr->list, &ftrace_trace_arrays);
7809
7810	mutex_unlock(&trace_types_lock);
7811	mutex_unlock(&event_mutex);
7812
7813	return 0;
7814
7815 out_free_tr:
7816	free_trace_buffers(tr);
7817	free_cpumask_var(tr->tracing_cpumask);
7818	kfree(tr->name);
7819	kfree(tr);
7820
7821 out_unlock:
7822	mutex_unlock(&trace_types_lock);
7823	mutex_unlock(&event_mutex);
7824
7825	return ret;
7826
7827}
7828
7829static int instance_rmdir(const char *name)
7830{
7831	struct trace_array *tr;
7832	int found = 0;
7833	int ret;
7834	int i;
7835
7836	mutex_lock(&event_mutex);
7837	mutex_lock(&trace_types_lock);
7838
7839	ret = -ENODEV;
7840	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7841		if (tr->name && strcmp(tr->name, name) == 0) {
7842			found = 1;
7843			break;
7844		}
7845	}
7846	if (!found)
7847		goto out_unlock;
7848
7849	ret = -EBUSY;
7850	if (tr->ref || (tr->current_trace && tr->current_trace->ref))
7851		goto out_unlock;
7852
7853	list_del(&tr->list);
7854
7855	/* Disable all the flags that were enabled coming in */
7856	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
7857		if ((1 << i) & ZEROED_TRACE_FLAGS)
7858			set_tracer_flag(tr, 1 << i, 0);
7859	}
7860
7861	tracing_set_nop(tr);
7862	clear_ftrace_function_probes(tr);
7863	event_trace_del_tracer(tr);
7864	ftrace_clear_pids(tr);
7865	ftrace_destroy_function_files(tr);
7866	tracefs_remove_recursive(tr->dir);
7867	free_trace_buffers(tr);
7868
7869	for (i = 0; i < tr->nr_topts; i++) {
7870		kfree(tr->topts[i].topts);
7871	}
7872	kfree(tr->topts);
7873
7874	free_cpumask_var(tr->tracing_cpumask);
7875	kfree(tr->name);
7876	kfree(tr);
7877
7878	ret = 0;
7879
7880 out_unlock:
7881	mutex_unlock(&trace_types_lock);
7882	mutex_unlock(&event_mutex);
7883
7884	return ret;
7885}
7886
7887static __init void create_trace_instances(struct dentry *d_tracer)
7888{
7889	trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
7890							 instance_mkdir,
7891							 instance_rmdir);
7892	if (WARN_ON(!trace_instance_dir))
7893		return;
7894}
7895
7896static void
7897init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
7898{
7899	int cpu;
7900
7901	trace_create_file("available_tracers", 0444, d_tracer,
7902			tr, &show_traces_fops);
7903
7904	trace_create_file("current_tracer", 0644, d_tracer,
7905			tr, &set_tracer_fops);
7906
7907	trace_create_file("tracing_cpumask", 0644, d_tracer,
7908			  tr, &tracing_cpumask_fops);
7909
7910	trace_create_file("trace_options", 0644, d_tracer,
7911			  tr, &tracing_iter_fops);
7912
7913	trace_create_file("trace", 0644, d_tracer,
7914			  tr, &tracing_fops);
7915
7916	trace_create_file("trace_pipe", 0444, d_tracer,
7917			  tr, &tracing_pipe_fops);
7918
7919	trace_create_file("buffer_size_kb", 0644, d_tracer,
7920			  tr, &tracing_entries_fops);
7921
7922	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
7923			  tr, &tracing_total_entries_fops);
7924
7925	trace_create_file("free_buffer", 0200, d_tracer,
7926			  tr, &tracing_free_buffer_fops);
7927
7928	trace_create_file("trace_marker", 0220, d_tracer,
7929			  tr, &tracing_mark_fops);
7930
7931	trace_create_file("trace_marker_raw", 0220, d_tracer,
7932			  tr, &tracing_mark_raw_fops);
7933
7934	trace_create_file("trace_clock", 0644, d_tracer, tr,
7935			  &trace_clock_fops);
7936
7937	trace_create_file("tracing_on", 0644, d_tracer,
7938			  tr, &rb_simple_fops);
7939
7940	trace_create_file("timestamp_mode", 0444, d_tracer, tr,
7941			  &trace_time_stamp_mode_fops);
7942
7943	create_trace_options_dir(tr);
7944
7945#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7946	trace_create_file("tracing_max_latency", 0644, d_tracer,
7947			&tr->max_latency, &tracing_max_lat_fops);
7948#endif
7949
7950	if (ftrace_create_function_files(tr, d_tracer))
7951		WARN(1, "Could not allocate function filter files");
7952
7953#ifdef CONFIG_TRACER_SNAPSHOT
7954	trace_create_file("snapshot", 0644, d_tracer,
7955			  tr, &snapshot_fops);
7956#endif
7957
7958	for_each_tracing_cpu(cpu)
7959		tracing_init_tracefs_percpu(tr, cpu);
7960
7961	ftrace_init_tracefs(tr, d_tracer);
7962}
7963
7964static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
7965{
7966	struct vfsmount *mnt;
7967	struct file_system_type *type;
7968
7969	/*
7970	 * To maintain backward compatibility for tools that mount
7971	 * debugfs to get to the tracing facility, tracefs is automatically
7972	 * mounted to the debugfs/tracing directory.
7973	 */
7974	type = get_fs_type("tracefs");
7975	if (!type)
7976		return NULL;
7977	mnt = vfs_submount(mntpt, type, "tracefs", NULL);
7978	put_filesystem(type);
7979	if (IS_ERR(mnt))
7980		return NULL;
7981	mntget(mnt);
7982
7983	return mnt;
7984}
7985
7986/**
7987 * tracing_init_dentry - initialize top level trace array
7988 *
7989 * This is called when creating files or directories in the tracing
7990 * directory. It is called via fs_initcall() by any of the boot up code
7991 * and expects to return the dentry of the top level tracing directory.
7992 */
7993struct dentry *tracing_init_dentry(void)
7994{
7995	struct trace_array *tr = &global_trace;
7996
7997	/* The top level trace array uses  NULL as parent */
7998	if (tr->dir)
7999		return NULL;
8000
8001	if (WARN_ON(!tracefs_initialized()) ||
8002		(IS_ENABLED(CONFIG_DEBUG_FS) &&
8003		 WARN_ON(!debugfs_initialized())))
8004		return ERR_PTR(-ENODEV);
8005
8006	/*
8007	 * As there may still be users that expect the tracing
8008	 * files to exist in debugfs/tracing, we must automount
8009	 * the tracefs file system there, so older tools still
8010	 * work with the newer kerenl.
8011	 */
8012	tr->dir = debugfs_create_automount("tracing", NULL,
8013					   trace_automount, NULL);
8014	if (!tr->dir) {
8015		pr_warn_once("Could not create debugfs directory 'tracing'\n");
8016		return ERR_PTR(-ENOMEM);
8017	}
8018
8019	return NULL;
8020}
8021
8022extern struct trace_eval_map *__start_ftrace_eval_maps[];
8023extern struct trace_eval_map *__stop_ftrace_eval_maps[];
8024
8025static void __init trace_eval_init(void)
8026{
8027	int len;
8028
8029	len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
8030	trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
8031}
8032
8033#ifdef CONFIG_MODULES
8034static void trace_module_add_evals(struct module *mod)
8035{
8036	if (!mod->num_trace_evals)
8037		return;
8038
8039	/*
8040	 * Modules with bad taint do not have events created, do
8041	 * not bother with enums either.
8042	 */
8043	if (trace_module_has_bad_taint(mod))
8044		return;
8045
8046	trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
8047}
8048
8049#ifdef CONFIG_TRACE_EVAL_MAP_FILE
8050static void trace_module_remove_evals(struct module *mod)
8051{
8052	union trace_eval_map_item *map;
8053	union trace_eval_map_item **last = &trace_eval_maps;
8054
8055	if (!mod->num_trace_evals)
8056		return;
8057
8058	mutex_lock(&trace_eval_mutex);
8059
8060	map = trace_eval_maps;
8061
8062	while (map) {
8063		if (map->head.mod == mod)
8064			break;
8065		map = trace_eval_jmp_to_tail(map);
8066		last = &map->tail.next;
8067		map = map->tail.next;
8068	}
8069	if (!map)
8070		goto out;
8071
8072	*last = trace_eval_jmp_to_tail(map)->tail.next;
8073	kfree(map);
8074 out:
8075	mutex_unlock(&trace_eval_mutex);
8076}
8077#else
8078static inline void trace_module_remove_evals(struct module *mod) { }
8079#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
8080
8081static int trace_module_notify(struct notifier_block *self,
8082			       unsigned long val, void *data)
8083{
8084	struct module *mod = data;
8085
8086	switch (val) {
8087	case MODULE_STATE_COMING:
8088		trace_module_add_evals(mod);
8089		break;
8090	case MODULE_STATE_GOING:
8091		trace_module_remove_evals(mod);
8092		break;
8093	}
8094
8095	return 0;
8096}
8097
8098static struct notifier_block trace_module_nb = {
8099	.notifier_call = trace_module_notify,
8100	.priority = 0,
8101};
8102#endif /* CONFIG_MODULES */
8103
8104static __init int tracer_init_tracefs(void)
8105{
8106	struct dentry *d_tracer;
8107
8108	trace_access_lock_init();
8109
8110	d_tracer = tracing_init_dentry();
8111	if (IS_ERR(d_tracer))
8112		return 0;
8113
8114	init_tracer_tracefs(&global_trace, d_tracer);
8115	ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
8116
8117	trace_create_file("tracing_thresh", 0644, d_tracer,
8118			&global_trace, &tracing_thresh_fops);
8119
8120	trace_create_file("README", 0444, d_tracer,
8121			NULL, &tracing_readme_fops);
8122
8123	trace_create_file("saved_cmdlines", 0444, d_tracer,
8124			NULL, &tracing_saved_cmdlines_fops);
8125
8126	trace_create_file("saved_cmdlines_size", 0644, d_tracer,
8127			  NULL, &tracing_saved_cmdlines_size_fops);
8128
8129	trace_create_file("saved_tgids", 0444, d_tracer,
8130			NULL, &tracing_saved_tgids_fops);
8131
8132	trace_eval_init();
8133
8134	trace_create_eval_file(d_tracer);
8135
8136#ifdef CONFIG_MODULES
8137	register_module_notifier(&trace_module_nb);
8138#endif
8139
8140#ifdef CONFIG_DYNAMIC_FTRACE
8141	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
8142			&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
8143#endif
8144
8145	create_trace_instances(d_tracer);
8146
8147	update_tracer_options(&global_trace);
8148
8149	return 0;
8150}
8151
8152static int trace_panic_handler(struct notifier_block *this,
8153			       unsigned long event, void *unused)
8154{
8155	if (ftrace_dump_on_oops)
8156		ftrace_dump(ftrace_dump_on_oops);
8157	return NOTIFY_OK;
8158}
8159
8160static struct notifier_block trace_panic_notifier = {
8161	.notifier_call  = trace_panic_handler,
8162	.next           = NULL,
8163	.priority       = 150   /* priority: INT_MAX >= x >= 0 */
8164};
8165
8166static int trace_die_handler(struct notifier_block *self,
8167			     unsigned long val,
8168			     void *data)
8169{
8170	switch (val) {
8171	case DIE_OOPS:
8172		if (ftrace_dump_on_oops)
8173			ftrace_dump(ftrace_dump_on_oops);
8174		break;
8175	default:
8176		break;
8177	}
8178	return NOTIFY_OK;
8179}
8180
8181static struct notifier_block trace_die_notifier = {
8182	.notifier_call = trace_die_handler,
8183	.priority = 200
8184};
8185
8186/*
8187 * printk is set to max of 1024, we really don't need it that big.
8188 * Nothing should be printing 1000 characters anyway.
8189 */
8190#define TRACE_MAX_PRINT		1000
8191
8192/*
8193 * Define here KERN_TRACE so that we have one place to modify
8194 * it if we decide to change what log level the ftrace dump
8195 * should be at.
8196 */
8197#define KERN_TRACE		KERN_EMERG
8198
8199void
8200trace_printk_seq(struct trace_seq *s)
8201{
8202	/* Probably should print a warning here. */
8203	if (s->seq.len >= TRACE_MAX_PRINT)
8204		s->seq.len = TRACE_MAX_PRINT;
8205
8206	/*
8207	 * More paranoid code. Although the buffer size is set to
8208	 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
8209	 * an extra layer of protection.
8210	 */
8211	if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
8212		s->seq.len = s->seq.size - 1;
8213
8214	/* should be zero ended, but we are paranoid. */
8215	s->buffer[s->seq.len] = 0;
8216
8217	printk(KERN_TRACE "%s", s->buffer);
8218
8219	trace_seq_init(s);
8220}
8221
8222void trace_init_global_iter(struct trace_iterator *iter)
8223{
8224	iter->tr = &global_trace;
8225	iter->trace = iter->tr->current_trace;
8226	iter->cpu_file = RING_BUFFER_ALL_CPUS;
8227	iter->trace_buffer = &global_trace.trace_buffer;
8228
8229	if (iter->trace && iter->trace->open)
8230		iter->trace->open(iter);
8231
8232	/* Annotate start of buffers if we had overruns */
8233	if (ring_buffer_overruns(iter->trace_buffer->buffer))
8234		iter->iter_flags |= TRACE_FILE_ANNOTATE;
8235
8236	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
8237	if (trace_clocks[iter->tr->clock_id].in_ns)
8238		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
8239}
8240
8241void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
8242{
8243	/* use static because iter can be a bit big for the stack */
8244	static struct trace_iterator iter;
8245	static atomic_t dump_running;
8246	struct trace_array *tr = &global_trace;
8247	unsigned int old_userobj;
8248	unsigned long flags;
8249	int cnt = 0, cpu;
8250
8251	/* Only allow one dump user at a time. */
8252	if (atomic_inc_return(&dump_running) != 1) {
8253		atomic_dec(&dump_running);
8254		return;
8255	}
8256
8257	/*
8258	 * Always turn off tracing when we dump.
8259	 * We don't need to show trace output of what happens
8260	 * between multiple crashes.
8261	 *
8262	 * If the user does a sysrq-z, then they can re-enable
8263	 * tracing with echo 1 > tracing_on.
8264	 */
8265	tracing_off();
8266
8267	local_irq_save(flags);
8268
8269	/* Simulate the iterator */
8270	trace_init_global_iter(&iter);
8271
8272	for_each_tracing_cpu(cpu) {
8273		atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
8274	}
8275
8276	old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
8277
8278	/* don't look at user memory in panic mode */
8279	tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
8280
8281	switch (oops_dump_mode) {
8282	case DUMP_ALL:
8283		iter.cpu_file = RING_BUFFER_ALL_CPUS;
8284		break;
8285	case DUMP_ORIG:
8286		iter.cpu_file = raw_smp_processor_id();
8287		break;
8288	case DUMP_NONE:
8289		goto out_enable;
8290	default:
8291		printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
8292		iter.cpu_file = RING_BUFFER_ALL_CPUS;
8293	}
8294
8295	printk(KERN_TRACE "Dumping ftrace buffer:\n");
8296
8297	/* Did function tracer already get disabled? */
8298	if (ftrace_is_dead()) {
8299		printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
8300		printk("#          MAY BE MISSING FUNCTION EVENTS\n");
8301	}
8302
8303	/*
8304	 * We need to stop all tracing on all CPUS to read the
8305	 * the next buffer. This is a bit expensive, but is
8306	 * not done often. We fill all what we can read,
8307	 * and then release the locks again.
8308	 */
8309
8310	while (!trace_empty(&iter)) {
8311
8312		if (!cnt)
8313			printk(KERN_TRACE "---------------------------------\n");
8314
8315		cnt++;
8316
8317		/* reset all but tr, trace, and overruns */
8318		memset(&iter.seq, 0,
8319		       sizeof(struct trace_iterator) -
8320		       offsetof(struct trace_iterator, seq));
8321		iter.iter_flags |= TRACE_FILE_LAT_FMT;
8322		iter.pos = -1;
8323
8324		if (trace_find_next_entry_inc(&iter) != NULL) {
8325			int ret;
8326
8327			ret = print_trace_line(&iter);
8328			if (ret != TRACE_TYPE_NO_CONSUME)
8329				trace_consume(&iter);
8330		}
8331		touch_nmi_watchdog();
8332
8333		trace_printk_seq(&iter.seq);
8334	}
8335
8336	if (!cnt)
8337		printk(KERN_TRACE "   (ftrace buffer empty)\n");
8338	else
8339		printk(KERN_TRACE "---------------------------------\n");
8340
8341 out_enable:
8342	tr->trace_flags |= old_userobj;
8343
8344	for_each_tracing_cpu(cpu) {
8345		atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
8346	}
8347 	atomic_dec(&dump_running);
8348	local_irq_restore(flags);
8349}
8350EXPORT_SYMBOL_GPL(ftrace_dump);
8351
8352int trace_run_command(const char *buf, int (*createfn)(int, char **))
8353{
8354	char **argv;
8355	int argc, ret;
8356
8357	argc = 0;
8358	ret = 0;
8359	argv = argv_split(GFP_KERNEL, buf, &argc);
8360	if (!argv)
8361		return -ENOMEM;
8362
8363	if (argc)
8364		ret = createfn(argc, argv);
8365
8366	argv_free(argv);
8367
8368	return ret;
8369}
8370
8371#define WRITE_BUFSIZE  4096
8372
8373ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
8374				size_t count, loff_t *ppos,
8375				int (*createfn)(int, char **))
8376{
8377	char *kbuf, *buf, *tmp;
8378	int ret = 0;
8379	size_t done = 0;
8380	size_t size;
8381
8382	kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
8383	if (!kbuf)
8384		return -ENOMEM;
8385
8386	while (done < count) {
8387		size = count - done;
8388
8389		if (size >= WRITE_BUFSIZE)
8390			size = WRITE_BUFSIZE - 1;
8391
8392		if (copy_from_user(kbuf, buffer + done, size)) {
8393			ret = -EFAULT;
8394			goto out;
8395		}
8396		kbuf[size] = '\0';
8397		buf = kbuf;
8398		do {
8399			tmp = strchr(buf, '\n');
8400			if (tmp) {
8401				*tmp = '\0';
8402				size = tmp - buf + 1;
8403			} else {
8404				size = strlen(buf);
8405				if (done + size < count) {
8406					if (buf != kbuf)
8407						break;
8408					/* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
8409					pr_warn("Line length is too long: Should be less than %d\n",
8410						WRITE_BUFSIZE - 2);
8411					ret = -EINVAL;
8412					goto out;
8413				}
8414			}
8415			done += size;
8416
8417			/* Remove comments */
8418			tmp = strchr(buf, '#');
8419
8420			if (tmp)
8421				*tmp = '\0';
8422
8423			ret = trace_run_command(buf, createfn);
8424			if (ret)
8425				goto out;
8426			buf += size;
8427
8428		} while (done < count);
8429	}
8430	ret = done;
8431
8432out:
8433	kfree(kbuf);
8434
8435	return ret;
8436}
8437
8438__init static int tracer_alloc_buffers(void)
8439{
8440	int ring_buf_size;
8441	int ret = -ENOMEM;
8442
8443	/*
8444	 * Make sure we don't accidently add more trace options
8445	 * than we have bits for.
8446	 */
8447	BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
8448
8449	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
8450		goto out;
8451
8452	if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
8453		goto out_free_buffer_mask;
8454
8455	/* Only allocate trace_printk buffers if a trace_printk exists */
8456	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
8457		/* Must be called before global_trace.buffer is allocated */
8458		trace_printk_init_buffers();
8459
8460	/* To save memory, keep the ring buffer size to its minimum */
8461	if (ring_buffer_expanded)
8462		ring_buf_size = trace_buf_size;
8463	else
8464		ring_buf_size = 1;
8465
8466	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
8467	cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
8468
8469	raw_spin_lock_init(&global_trace.start_lock);
8470
8471	/*
8472	 * The prepare callbacks allocates some memory for the ring buffer. We
8473	 * don't free the buffer if the if the CPU goes down. If we were to free
8474	 * the buffer, then the user would lose any trace that was in the
8475	 * buffer. The memory will be removed once the "instance" is removed.
8476	 */
8477	ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
8478				      "trace/RB:preapre", trace_rb_cpu_prepare,
8479				      NULL);
8480	if (ret < 0)
8481		goto out_free_cpumask;
8482	/* Used for event triggers */
8483	ret = -ENOMEM;
8484	temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
8485	if (!temp_buffer)
8486		goto out_rm_hp_state;
8487
8488	if (trace_create_savedcmd() < 0)
8489		goto out_free_temp_buffer;
8490
8491	/* TODO: make the number of buffers hot pluggable with CPUS */
8492	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
8493		printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
8494		WARN_ON(1);
8495		goto out_free_savedcmd;
8496	}
8497
8498	if (global_trace.buffer_disabled)
8499		tracing_off();
8500
8501	if (trace_boot_clock) {
8502		ret = tracing_set_clock(&global_trace, trace_boot_clock);
8503		if (ret < 0)
8504			pr_warn("Trace clock %s not defined, going back to default\n",
8505				trace_boot_clock);
8506	}
8507
8508	/*
8509	 * register_tracer() might reference current_trace, so it
8510	 * needs to be set before we register anything. This is
8511	 * just a bootstrap of current_trace anyway.
8512	 */
8513	global_trace.current_trace = &nop_trace;
8514
8515	global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8516
8517	ftrace_init_global_array_ops(&global_trace);
8518
8519	init_trace_flags_index(&global_trace);
8520
8521	register_tracer(&nop_trace);
8522
8523	/* Function tracing may start here (via kernel command line) */
8524	init_function_trace();
8525
8526	/* All seems OK, enable tracing */
8527	tracing_disabled = 0;
8528
8529	atomic_notifier_chain_register(&panic_notifier_list,
8530				       &trace_panic_notifier);
8531
8532	register_die_notifier(&trace_die_notifier);
8533
8534	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
8535
8536	INIT_LIST_HEAD(&global_trace.systems);
8537	INIT_LIST_HEAD(&global_trace.events);
8538	INIT_LIST_HEAD(&global_trace.hist_vars);
8539	list_add(&global_trace.list, &ftrace_trace_arrays);
8540
8541	apply_trace_boot_options();
8542
8543	register_snapshot_cmd();
8544
8545	return 0;
8546
8547out_free_savedcmd:
8548	free_saved_cmdlines_buffer(savedcmd);
8549out_free_temp_buffer:
8550	ring_buffer_free(temp_buffer);
8551out_rm_hp_state:
8552	cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
8553out_free_cpumask:
8554	free_cpumask_var(global_trace.tracing_cpumask);
8555out_free_buffer_mask:
8556	free_cpumask_var(tracing_buffer_mask);
8557out:
8558	return ret;
8559}
8560
8561void __init early_trace_init(void)
8562{
8563	if (tracepoint_printk) {
8564		tracepoint_print_iter =
8565			kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
8566		if (WARN_ON(!tracepoint_print_iter))
8567			tracepoint_printk = 0;
8568		else
8569			static_key_enable(&tracepoint_printk_key.key);
8570	}
8571	tracer_alloc_buffers();
8572}
8573
8574void __init trace_init(void)
8575{
8576	trace_event_init();
8577}
8578
8579__init static int clear_boot_tracer(void)
8580{
8581	/*
8582	 * The default tracer at boot buffer is an init section.
8583	 * This function is called in lateinit. If we did not
8584	 * find the boot tracer, then clear it out, to prevent
8585	 * later registration from accessing the buffer that is
8586	 * about to be freed.
8587	 */
8588	if (!default_bootup_tracer)
8589		return 0;
8590
8591	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
8592	       default_bootup_tracer);
8593	default_bootup_tracer = NULL;
8594
8595	return 0;
8596}
8597
8598fs_initcall(tracer_init_tracefs);
8599late_initcall_sync(clear_boot_tracer);
8600
8601#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
8602__init static int tracing_set_default_clock(void)
8603{
8604	/* sched_clock_stable() is determined in late_initcall */
8605	if (!trace_boot_clock && !sched_clock_stable()) {
8606		printk(KERN_WARNING
8607		       "Unstable clock detected, switching default tracing clock to \"global\"\n"
8608		       "If you want to keep using the local clock, then add:\n"
8609		       "  \"trace_clock=local\"\n"
8610		       "on the kernel command line\n");
8611		tracing_set_clock(&global_trace, "global");
8612	}
8613
8614	return 0;
8615}
8616late_initcall_sync(tracing_set_default_clock);
8617#endif