1/*
   2 * Infrastructure for profiling code inserted by 'gcc -pg'.
   3 *
   4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6 *
   7 * Originally ported from the -rt patch by:
   8 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9 *
  10 * Based on code in the latency_tracer, that is:
  11 *
  12 *  Copyright (C) 2004-2006 Ingo Molnar
  13 *  Copyright (C) 2004 Nadia Yvette Chambers
  14 */
  15
  16#include <linux/stop_machine.h>
  17#include <linux/clocksource.h>
 
  18#include <linux/kallsyms.h>
 
  19#include <linux/seq_file.h>
  20#include <linux/suspend.h>
  21#include <linux/debugfs.h>
  22#include <linux/hardirq.h>
  23#include <linux/kthread.h>
  24#include <linux/uaccess.h>
  25#include <linux/bsearch.h>
  26#include <linux/module.h>
  27#include <linux/ftrace.h>
  28#include <linux/sysctl.h>
  29#include <linux/slab.h>
  30#include <linux/ctype.h>
  31#include <linux/sort.h>
  32#include <linux/list.h>
  33#include <linux/hash.h>
  34#include <linux/rcupdate.h>
 
  35
  36#include <trace/events/sched.h>
  37
 
  38#include <asm/setup.h>
  39
 
  40#include "trace_output.h"
  41#include "trace_stat.h"
  42
 
 
  43#define FTRACE_WARN_ON(cond)			\
  44	({					\
  45		int ___r = cond;		\
  46		if (WARN_ON(___r))		\
  47			ftrace_kill();		\
  48		___r;				\
  49	})
  50
  51#define FTRACE_WARN_ON_ONCE(cond)		\
  52	({					\
  53		int ___r = cond;		\
  54		if (WARN_ON_ONCE(___r))		\
  55			ftrace_kill();		\
  56		___r;				\
  57	})
  58
  59/* hash bits for specific function selection */
  60#define FTRACE_HASH_BITS 7
  61#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  62#define FTRACE_HASH_DEFAULT_BITS 10
  63#define FTRACE_HASH_MAX_BITS 12
  64
  65#define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL)
  66
  67#ifdef CONFIG_DYNAMIC_FTRACE
  68#define INIT_REGEX_LOCK(opsname)	\
  69	.regex_lock	= __MUTEX_INITIALIZER(opsname.regex_lock),
 
  70#else
  71#define INIT_REGEX_LOCK(opsname)
  72#endif
  73
  74static struct ftrace_ops ftrace_list_end __read_mostly = {
 
 
 
 
 
  75	.func		= ftrace_stub,
  76	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
 
  77};
  78
  79/* ftrace_enabled is a method to turn ftrace on or off */
  80int ftrace_enabled __read_mostly;
  81static int last_ftrace_enabled;
  82
  83/* Quick disabling of function tracer. */
  84int function_trace_stop __read_mostly;
  85
  86/* Current function tracing op */
  87struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
  88/* What to set function_trace_op to */
  89static struct ftrace_ops *set_function_trace_op;
  90
  91/* List for set_ftrace_pid's pids. */
  92LIST_HEAD(ftrace_pids);
  93struct ftrace_pid {
  94	struct list_head list;
  95	struct pid *pid;
  96};
 
 
 
 
 
 
 
  97
  98/*
  99 * ftrace_disabled is set when an anomaly is discovered.
 100 * ftrace_disabled is much stronger than ftrace_enabled.
 101 */
 102static int ftrace_disabled __read_mostly;
 103
 104static DEFINE_MUTEX(ftrace_lock);
 105
 106static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
 107static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
 108static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
 109ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
 110ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
 111static struct ftrace_ops global_ops;
 112static struct ftrace_ops control_ops;
 113
 114#if ARCH_SUPPORTS_FTRACE_OPS
 115static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
 116				 struct ftrace_ops *op, struct pt_regs *regs);
 117#else
 118/* See comment below, where ftrace_ops_list_func is defined */
 119static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
 120#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
 121#endif
 122
 123/*
 124 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
 125 * can use rcu_dereference_raw_notrace() is that elements removed from this list
 126 * are simply leaked, so there is no need to interact with a grace-period
 127 * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
 128 * concurrent insertions into the ftrace_global_list.
 129 *
 130 * Silly Alpha and silly pointer-speculation compiler optimizations!
 131 */
 132#define do_for_each_ftrace_op(op, list)			\
 133	op = rcu_dereference_raw_notrace(list);			\
 134	do
 135
 136/*
 137 * Optimized for just a single item in the list (as that is the normal case).
 138 */
 139#define while_for_each_ftrace_op(op)				\
 140	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
 141	       unlikely((op) != &ftrace_list_end))
 142
 143static inline void ftrace_ops_init(struct ftrace_ops *ops)
 144{
 145#ifdef CONFIG_DYNAMIC_FTRACE
 146	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
 147		mutex_init(&ops->regex_lock);
 
 148		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
 149	}
 150#endif
 151}
 152
 153/**
 154 * ftrace_nr_registered_ops - return number of ops registered
 155 *
 156 * Returns the number of ftrace_ops registered and tracing functions
 157 */
 158int ftrace_nr_registered_ops(void)
 159{
 160	struct ftrace_ops *ops;
 161	int cnt = 0;
 162
 163	mutex_lock(&ftrace_lock);
 164
 165	for (ops = ftrace_ops_list;
 166	     ops != &ftrace_list_end; ops = ops->next)
 167		cnt++;
 168
 169	mutex_unlock(&ftrace_lock);
 170
 171	return cnt;
 172}
 173
 174static void
 175ftrace_global_list_func(unsigned long ip, unsigned long parent_ip,
 176			struct ftrace_ops *op, struct pt_regs *regs)
 177{
 178	int bit;
 179
 180	bit = trace_test_and_set_recursion(TRACE_GLOBAL_START, TRACE_GLOBAL_MAX);
 181	if (bit < 0)
 182		return;
 183
 184	do_for_each_ftrace_op(op, ftrace_global_list) {
 185		op->func(ip, parent_ip, op, regs);
 186	} while_for_each_ftrace_op(op);
 187
 188	trace_clear_recursion(bit);
 189}
 190
 191static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
 192			    struct ftrace_ops *op, struct pt_regs *regs)
 193{
 194	if (!test_tsk_trace_trace(current))
 195		return;
 196
 197	ftrace_pid_function(ip, parent_ip, op, regs);
 198}
 199
 200static void set_ftrace_pid_function(ftrace_func_t func)
 201{
 202	/* do not set ftrace_pid_function to itself! */
 203	if (func != ftrace_pid_func)
 204		ftrace_pid_function = func;
 205}
 206
 207/**
 208 * clear_ftrace_function - reset the ftrace function
 209 *
 210 * This NULLs the ftrace function and in essence stops
 211 * tracing.  There may be lag
 212 */
 213void clear_ftrace_function(void)
 214{
 215	ftrace_trace_function = ftrace_stub;
 216	ftrace_pid_function = ftrace_stub;
 217}
 218
 219static void control_ops_disable_all(struct ftrace_ops *ops)
 220{
 221	int cpu;
 222
 223	for_each_possible_cpu(cpu)
 224		*per_cpu_ptr(ops->disabled, cpu) = 1;
 225}
 226
 227static int control_ops_alloc(struct ftrace_ops *ops)
 228{
 229	int __percpu *disabled;
 230
 231	disabled = alloc_percpu(int);
 232	if (!disabled)
 233		return -ENOMEM;
 234
 235	ops->disabled = disabled;
 236	control_ops_disable_all(ops);
 237	return 0;
 238}
 239
 240static void update_global_ops(void)
 241{
 242	ftrace_func_t func = ftrace_global_list_func;
 243	void *private = NULL;
 244
 245	/* The list has its own recursion protection. */
 246	global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE;
 247
 248	/*
 249	 * If there's only one function registered, then call that
 250	 * function directly. Otherwise, we need to iterate over the
 251	 * registered callers.
 252	 */
 253	if (ftrace_global_list == &ftrace_list_end ||
 254	    ftrace_global_list->next == &ftrace_list_end) {
 255		func = ftrace_global_list->func;
 256		private = ftrace_global_list->private;
 257		/*
 258		 * As we are calling the function directly.
 259		 * If it does not have recursion protection,
 260		 * the function_trace_op needs to be updated
 261		 * accordingly.
 262		 */
 263		if (!(ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE))
 264			global_ops.flags &= ~FTRACE_OPS_FL_RECURSION_SAFE;
 265	}
 266
 267	/* If we filter on pids, update to use the pid function */
 268	if (!list_empty(&ftrace_pids)) {
 269		set_ftrace_pid_function(func);
 270		func = ftrace_pid_func;
 271	}
 272
 273	global_ops.func = func;
 274	global_ops.private = private;
 275}
 276
 277static void ftrace_sync(struct work_struct *work)
 278{
 279	/*
 280	 * This function is just a stub to implement a hard force
 281	 * of synchronize_sched(). This requires synchronizing
 282	 * tasks even in userspace and idle.
 283	 *
 284	 * Yes, function tracing is rude.
 285	 */
 286}
 
 
 287
 288static void ftrace_sync_ipi(void *data)
 289{
 290	/* Probably not needed, but do it anyway */
 291	smp_rmb();
 292}
 293
 294#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 295static void update_function_graph_func(void);
 296#else
 297static inline void update_function_graph_func(void) { }
 298#endif
 299
 300static void update_ftrace_function(void)
 301{
 302	ftrace_func_t func;
 303
 304	update_global_ops();
 
 
 
 
 
 
 
 
 
 
 305
 306	/*
 307	 * If we are at the end of the list and this ops is
 308	 * recursion safe and not dynamic and the arch supports passing ops,
 309	 * then have the mcount trampoline call the function directly.
 310	 */
 311	if (ftrace_ops_list == &ftrace_list_end ||
 312	    (ftrace_ops_list->next == &ftrace_list_end &&
 313	     !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) &&
 314	     (ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) &&
 315	     !FTRACE_FORCE_LIST_FUNC)) {
 316		/* Set the ftrace_ops that the arch callback uses */
 317		if (ftrace_ops_list == &global_ops)
 318			set_function_trace_op = ftrace_global_list;
 319		else
 320			set_function_trace_op = ftrace_ops_list;
 321		func = ftrace_ops_list->func;
 322	} else {
 323		/* Just use the default ftrace_ops */
 324		set_function_trace_op = &ftrace_list_end;
 325		func = ftrace_ops_list_func;
 326	}
 327
 
 
 328	/* If there's no change, then do nothing more here */
 329	if (ftrace_trace_function == func)
 330		return;
 331
 332	update_function_graph_func();
 333
 334	/*
 335	 * If we are using the list function, it doesn't care
 336	 * about the function_trace_ops.
 337	 */
 338	if (func == ftrace_ops_list_func) {
 339		ftrace_trace_function = func;
 340		/*
 341		 * Don't even bother setting function_trace_ops,
 342		 * it would be racy to do so anyway.
 343		 */
 344		return;
 345	}
 346
 347#ifndef CONFIG_DYNAMIC_FTRACE
 348	/*
 349	 * For static tracing, we need to be a bit more careful.
 350	 * The function change takes affect immediately. Thus,
 351	 * we need to coorditate the setting of the function_trace_ops
 352	 * with the setting of the ftrace_trace_function.
 353	 *
 354	 * Set the function to the list ops, which will call the
 355	 * function we want, albeit indirectly, but it handles the
 356	 * ftrace_ops and doesn't depend on function_trace_op.
 357	 */
 358	ftrace_trace_function = ftrace_ops_list_func;
 359	/*
 360	 * Make sure all CPUs see this. Yes this is slow, but static
 361	 * tracing is slow and nasty to have enabled.
 362	 */
 363	schedule_on_each_cpu(ftrace_sync);
 364	/* Now all cpus are using the list ops. */
 365	function_trace_op = set_function_trace_op;
 366	/* Make sure the function_trace_op is visible on all CPUs */
 367	smp_wmb();
 368	/* Nasty way to force a rmb on all cpus */
 369	smp_call_function(ftrace_sync_ipi, NULL, 1);
 370	/* OK, we are all set to update the ftrace_trace_function now! */
 371#endif /* !CONFIG_DYNAMIC_FTRACE */
 372
 373	ftrace_trace_function = func;
 374}
 375
 376static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 
 377{
 378	ops->next = *list;
 
 379	/*
 380	 * We are entering ops into the list but another
 381	 * CPU might be walking that list. We need to make sure
 382	 * the ops->next pointer is valid before another CPU sees
 383	 * the ops pointer included into the list.
 384	 */
 385	rcu_assign_pointer(*list, ops);
 386}
 387
 388static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 
 389{
 390	struct ftrace_ops **p;
 391
 392	/*
 393	 * If we are removing the last function, then simply point
 394	 * to the ftrace_stub.
 395	 */
 396	if (*list == ops && ops->next == &ftrace_list_end) {
 
 
 
 397		*list = &ftrace_list_end;
 398		return 0;
 399	}
 400
 401	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
 402		if (*p == ops)
 403			break;
 404
 405	if (*p != ops)
 406		return -1;
 407
 408	*p = (*p)->next;
 409	return 0;
 410}
 411
 412static void add_ftrace_list_ops(struct ftrace_ops **list,
 413				struct ftrace_ops *main_ops,
 414				struct ftrace_ops *ops)
 415{
 416	int first = *list == &ftrace_list_end;
 417	add_ftrace_ops(list, ops);
 418	if (first)
 419		add_ftrace_ops(&ftrace_ops_list, main_ops);
 420}
 421
 422static int remove_ftrace_list_ops(struct ftrace_ops **list,
 423				  struct ftrace_ops *main_ops,
 424				  struct ftrace_ops *ops)
 425{
 426	int ret = remove_ftrace_ops(list, ops);
 427	if (!ret && *list == &ftrace_list_end)
 428		ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
 429	return ret;
 430}
 431
 432static int __register_ftrace_function(struct ftrace_ops *ops)
 433{
 434	if (ops->flags & FTRACE_OPS_FL_DELETED)
 435		return -EINVAL;
 436
 437	if (FTRACE_WARN_ON(ops == &global_ops))
 438		return -EINVAL;
 439
 440	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
 441		return -EBUSY;
 442
 443	/* We don't support both control and global flags set. */
 444	if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)
 445		return -EINVAL;
 446
 447#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 448	/*
 449	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
 450	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
 451	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
 452	 */
 453	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
 454	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
 455		return -EINVAL;
 456
 457	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
 458		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
 459#endif
 
 
 460
 461	if (!core_kernel_data((unsigned long)ops))
 462		ops->flags |= FTRACE_OPS_FL_DYNAMIC;
 463
 464	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 465		add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops);
 466		ops->flags |= FTRACE_OPS_FL_ENABLED;
 467	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
 468		if (control_ops_alloc(ops))
 469			return -ENOMEM;
 470		add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
 471	} else
 472		add_ftrace_ops(&ftrace_ops_list, ops);
 473
 474	if (ftrace_enabled)
 475		update_ftrace_function();
 476
 477	return 0;
 478}
 479
 480static int __unregister_ftrace_function(struct ftrace_ops *ops)
 481{
 482	int ret;
 483
 484	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
 485		return -EBUSY;
 486
 487	if (FTRACE_WARN_ON(ops == &global_ops))
 488		return -EINVAL;
 489
 490	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 491		ret = remove_ftrace_list_ops(&ftrace_global_list,
 492					     &global_ops, ops);
 493		if (!ret)
 494			ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 495	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
 496		ret = remove_ftrace_list_ops(&ftrace_control_list,
 497					     &control_ops, ops);
 498	} else
 499		ret = remove_ftrace_ops(&ftrace_ops_list, ops);
 500
 501	if (ret < 0)
 502		return ret;
 503
 504	if (ftrace_enabled)
 505		update_ftrace_function();
 506
 
 
 507	return 0;
 508}
 509
 510static void ftrace_update_pid_func(void)
 511{
 
 
 512	/* Only do something if we are tracing something */
 513	if (ftrace_trace_function == ftrace_stub)
 514		return;
 515
 
 
 
 
 
 
 
 
 516	update_ftrace_function();
 517}
 518
 519#ifdef CONFIG_FUNCTION_PROFILER
 520struct ftrace_profile {
 521	struct hlist_node		node;
 522	unsigned long			ip;
 523	unsigned long			counter;
 524#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 525	unsigned long long		time;
 526	unsigned long long		time_squared;
 527#endif
 528};
 529
 530struct ftrace_profile_page {
 531	struct ftrace_profile_page	*next;
 532	unsigned long			index;
 533	struct ftrace_profile		records[];
 534};
 535
 536struct ftrace_profile_stat {
 537	atomic_t			disabled;
 538	struct hlist_head		*hash;
 539	struct ftrace_profile_page	*pages;
 540	struct ftrace_profile_page	*start;
 541	struct tracer_stat		stat;
 542};
 543
 544#define PROFILE_RECORDS_SIZE						\
 545	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 546
 547#define PROFILES_PER_PAGE					\
 548	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 549
 550static int ftrace_profile_enabled __read_mostly;
 551
 552/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 553static DEFINE_MUTEX(ftrace_profile_lock);
 554
 555static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 556
 557#define FTRACE_PROFILE_HASH_BITS 10
 558#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
 559
 560static void *
 561function_stat_next(void *v, int idx)
 562{
 563	struct ftrace_profile *rec = v;
 564	struct ftrace_profile_page *pg;
 565
 566	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 567
 568 again:
 569	if (idx != 0)
 570		rec++;
 571
 572	if ((void *)rec >= (void *)&pg->records[pg->index]) {
 573		pg = pg->next;
 574		if (!pg)
 575			return NULL;
 576		rec = &pg->records[0];
 577		if (!rec->counter)
 578			goto again;
 579	}
 580
 581	return rec;
 582}
 583
 584static void *function_stat_start(struct tracer_stat *trace)
 585{
 586	struct ftrace_profile_stat *stat =
 587		container_of(trace, struct ftrace_profile_stat, stat);
 588
 589	if (!stat || !stat->start)
 590		return NULL;
 591
 592	return function_stat_next(&stat->start->records[0], 0);
 593}
 594
 595#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 596/* function graph compares on total time */
 597static int function_stat_cmp(void *p1, void *p2)
 598{
 599	struct ftrace_profile *a = p1;
 600	struct ftrace_profile *b = p2;
 601
 602	if (a->time < b->time)
 603		return -1;
 604	if (a->time > b->time)
 605		return 1;
 606	else
 607		return 0;
 608}
 609#else
 610/* not function graph compares against hits */
 611static int function_stat_cmp(void *p1, void *p2)
 612{
 613	struct ftrace_profile *a = p1;
 614	struct ftrace_profile *b = p2;
 615
 616	if (a->counter < b->counter)
 617		return -1;
 618	if (a->counter > b->counter)
 619		return 1;
 620	else
 621		return 0;
 622}
 623#endif
 624
 625static int function_stat_headers(struct seq_file *m)
 626{
 627#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 628	seq_printf(m, "  Function                               "
 629		   "Hit    Time            Avg             s^2\n"
 630		      "  --------                               "
 631		   "---    ----            ---             ---\n");
 632#else
 633	seq_printf(m, "  Function                               Hit\n"
 634		      "  --------                               ---\n");
 635#endif
 636	return 0;
 637}
 638
 639static int function_stat_show(struct seq_file *m, void *v)
 640{
 641	struct ftrace_profile *rec = v;
 642	char str[KSYM_SYMBOL_LEN];
 643	int ret = 0;
 644#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 645	static struct trace_seq s;
 646	unsigned long long avg;
 647	unsigned long long stddev;
 648#endif
 649	mutex_lock(&ftrace_profile_lock);
 650
 651	/* we raced with function_profile_reset() */
 652	if (unlikely(rec->counter == 0)) {
 653		ret = -EBUSY;
 654		goto out;
 655	}
 656
 
 
 
 
 
 
 657	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 658	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 659
 660#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 661	seq_printf(m, "    ");
 662	avg = rec->time;
 663	do_div(avg, rec->counter);
 664
 665	/* Sample standard deviation (s^2) */
 666	if (rec->counter <= 1)
 667		stddev = 0;
 668	else {
 669		/*
 670		 * Apply Welford's method:
 671		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
 672		 */
 673		stddev = rec->counter * rec->time_squared -
 674			 rec->time * rec->time;
 675
 676		/*
 677		 * Divide only 1000 for ns^2 -> us^2 conversion.
 678		 * trace_print_graph_duration will divide 1000 again.
 679		 */
 680		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
 
 681	}
 682
 683	trace_seq_init(&s);
 684	trace_print_graph_duration(rec->time, &s);
 685	trace_seq_puts(&s, "    ");
 686	trace_print_graph_duration(avg, &s);
 687	trace_seq_puts(&s, "    ");
 688	trace_print_graph_duration(stddev, &s);
 689	trace_print_seq(m, &s);
 690#endif
 691	seq_putc(m, '\n');
 692out:
 693	mutex_unlock(&ftrace_profile_lock);
 694
 695	return ret;
 696}
 697
 698static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 699{
 700	struct ftrace_profile_page *pg;
 701
 702	pg = stat->pages = stat->start;
 703
 704	while (pg) {
 705		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 706		pg->index = 0;
 707		pg = pg->next;
 708	}
 709
 710	memset(stat->hash, 0,
 711	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 712}
 713
 714int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 715{
 716	struct ftrace_profile_page *pg;
 717	int functions;
 718	int pages;
 719	int i;
 720
 721	/* If we already allocated, do nothing */
 722	if (stat->pages)
 723		return 0;
 724
 725	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 726	if (!stat->pages)
 727		return -ENOMEM;
 728
 729#ifdef CONFIG_DYNAMIC_FTRACE
 730	functions = ftrace_update_tot_cnt;
 731#else
 732	/*
 733	 * We do not know the number of functions that exist because
 734	 * dynamic tracing is what counts them. With past experience
 735	 * we have around 20K functions. That should be more than enough.
 736	 * It is highly unlikely we will execute every function in
 737	 * the kernel.
 738	 */
 739	functions = 20000;
 740#endif
 741
 742	pg = stat->start = stat->pages;
 743
 744	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 745
 746	for (i = 1; i < pages; i++) {
 747		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 748		if (!pg->next)
 749			goto out_free;
 750		pg = pg->next;
 751	}
 752
 753	return 0;
 754
 755 out_free:
 756	pg = stat->start;
 757	while (pg) {
 758		unsigned long tmp = (unsigned long)pg;
 759
 760		pg = pg->next;
 761		free_page(tmp);
 762	}
 763
 764	stat->pages = NULL;
 765	stat->start = NULL;
 766
 767	return -ENOMEM;
 768}
 769
 770static int ftrace_profile_init_cpu(int cpu)
 771{
 772	struct ftrace_profile_stat *stat;
 773	int size;
 774
 775	stat = &per_cpu(ftrace_profile_stats, cpu);
 776
 777	if (stat->hash) {
 778		/* If the profile is already created, simply reset it */
 779		ftrace_profile_reset(stat);
 780		return 0;
 781	}
 782
 783	/*
 784	 * We are profiling all functions, but usually only a few thousand
 785	 * functions are hit. We'll make a hash of 1024 items.
 786	 */
 787	size = FTRACE_PROFILE_HASH_SIZE;
 788
 789	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 790
 791	if (!stat->hash)
 792		return -ENOMEM;
 793
 794	/* Preallocate the function profiling pages */
 795	if (ftrace_profile_pages_init(stat) < 0) {
 796		kfree(stat->hash);
 797		stat->hash = NULL;
 798		return -ENOMEM;
 799	}
 800
 801	return 0;
 802}
 803
 804static int ftrace_profile_init(void)
 805{
 806	int cpu;
 807	int ret = 0;
 808
 809	for_each_possible_cpu(cpu) {
 810		ret = ftrace_profile_init_cpu(cpu);
 811		if (ret)
 812			break;
 813	}
 814
 815	return ret;
 816}
 817
 818/* interrupts must be disabled */
 819static struct ftrace_profile *
 820ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 821{
 822	struct ftrace_profile *rec;
 823	struct hlist_head *hhd;
 824	unsigned long key;
 825
 826	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
 827	hhd = &stat->hash[key];
 828
 829	if (hlist_empty(hhd))
 830		return NULL;
 831
 832	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
 833		if (rec->ip == ip)
 834			return rec;
 835	}
 836
 837	return NULL;
 838}
 839
 840static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 841			       struct ftrace_profile *rec)
 842{
 843	unsigned long key;
 844
 845	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
 846	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 847}
 848
 849/*
 850 * The memory is already allocated, this simply finds a new record to use.
 851 */
 852static struct ftrace_profile *
 853ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 854{
 855	struct ftrace_profile *rec = NULL;
 856
 857	/* prevent recursion (from NMIs) */
 858	if (atomic_inc_return(&stat->disabled) != 1)
 859		goto out;
 860
 861	/*
 862	 * Try to find the function again since an NMI
 863	 * could have added it
 864	 */
 865	rec = ftrace_find_profiled_func(stat, ip);
 866	if (rec)
 867		goto out;
 868
 869	if (stat->pages->index == PROFILES_PER_PAGE) {
 870		if (!stat->pages->next)
 871			goto out;
 872		stat->pages = stat->pages->next;
 873	}
 874
 875	rec = &stat->pages->records[stat->pages->index++];
 876	rec->ip = ip;
 877	ftrace_add_profile(stat, rec);
 878
 879 out:
 880	atomic_dec(&stat->disabled);
 881
 882	return rec;
 883}
 884
 885static void
 886function_profile_call(unsigned long ip, unsigned long parent_ip,
 887		      struct ftrace_ops *ops, struct pt_regs *regs)
 888{
 889	struct ftrace_profile_stat *stat;
 890	struct ftrace_profile *rec;
 891	unsigned long flags;
 892
 893	if (!ftrace_profile_enabled)
 894		return;
 895
 896	local_irq_save(flags);
 897
 898	stat = &__get_cpu_var(ftrace_profile_stats);
 899	if (!stat->hash || !ftrace_profile_enabled)
 900		goto out;
 901
 902	rec = ftrace_find_profiled_func(stat, ip);
 903	if (!rec) {
 904		rec = ftrace_profile_alloc(stat, ip);
 905		if (!rec)
 906			goto out;
 907	}
 908
 909	rec->counter++;
 910 out:
 911	local_irq_restore(flags);
 912}
 913
 914#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 
 
 
 
 
 
 
 915static int profile_graph_entry(struct ftrace_graph_ent *trace)
 916{
 
 
 917	function_profile_call(trace->func, 0, NULL, NULL);
 
 
 
 
 
 
 
 
 
 918	return 1;
 919}
 920
 921static void profile_graph_return(struct ftrace_graph_ret *trace)
 922{
 
 923	struct ftrace_profile_stat *stat;
 924	unsigned long long calltime;
 925	struct ftrace_profile *rec;
 926	unsigned long flags;
 927
 928	local_irq_save(flags);
 929	stat = &__get_cpu_var(ftrace_profile_stats);
 930	if (!stat->hash || !ftrace_profile_enabled)
 931		goto out;
 932
 933	/* If the calltime was zero'd ignore it */
 934	if (!trace->calltime)
 935		goto out;
 936
 937	calltime = trace->rettime - trace->calltime;
 938
 939	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 940		int index;
 941
 942		index = trace->depth;
 943
 944		/* Append this call time to the parent time to subtract */
 945		if (index)
 946			current->ret_stack[index - 1].subtime += calltime;
 947
 948		if (current->ret_stack[index].subtime < calltime)
 949			calltime -= current->ret_stack[index].subtime;
 
 
 950		else
 951			calltime = 0;
 952	}
 953
 954	rec = ftrace_find_profiled_func(stat, trace->func);
 955	if (rec) {
 956		rec->time += calltime;
 957		rec->time_squared += calltime * calltime;
 958	}
 959
 960 out:
 961	local_irq_restore(flags);
 962}
 963
 
 
 
 
 
 964static int register_ftrace_profiler(void)
 965{
 966	return register_ftrace_graph(&profile_graph_return,
 967				     &profile_graph_entry);
 968}
 969
 970static void unregister_ftrace_profiler(void)
 971{
 972	unregister_ftrace_graph();
 973}
 974#else
 975static struct ftrace_ops ftrace_profile_ops __read_mostly = {
 976	.func		= function_profile_call,
 977	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
 978	INIT_REGEX_LOCK(ftrace_profile_ops)
 979};
 980
 981static int register_ftrace_profiler(void)
 982{
 983	return register_ftrace_function(&ftrace_profile_ops);
 984}
 985
 986static void unregister_ftrace_profiler(void)
 987{
 988	unregister_ftrace_function(&ftrace_profile_ops);
 989}
 990#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 991
 992static ssize_t
 993ftrace_profile_write(struct file *filp, const char __user *ubuf,
 994		     size_t cnt, loff_t *ppos)
 995{
 996	unsigned long val;
 997	int ret;
 998
 999	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1000	if (ret)
1001		return ret;
1002
1003	val = !!val;
1004
1005	mutex_lock(&ftrace_profile_lock);
1006	if (ftrace_profile_enabled ^ val) {
1007		if (val) {
1008			ret = ftrace_profile_init();
1009			if (ret < 0) {
1010				cnt = ret;
1011				goto out;
1012			}
1013
1014			ret = register_ftrace_profiler();
1015			if (ret < 0) {
1016				cnt = ret;
1017				goto out;
1018			}
1019			ftrace_profile_enabled = 1;
1020		} else {
1021			ftrace_profile_enabled = 0;
1022			/*
1023			 * unregister_ftrace_profiler calls stop_machine
1024			 * so this acts like an synchronize_sched.
1025			 */
1026			unregister_ftrace_profiler();
1027		}
1028	}
1029 out:
1030	mutex_unlock(&ftrace_profile_lock);
1031
1032	*ppos += cnt;
1033
1034	return cnt;
1035}
1036
1037static ssize_t
1038ftrace_profile_read(struct file *filp, char __user *ubuf,
1039		     size_t cnt, loff_t *ppos)
1040{
1041	char buf[64];		/* big enough to hold a number */
1042	int r;
1043
1044	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
1045	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1046}
1047
1048static const struct file_operations ftrace_profile_fops = {
1049	.open		= tracing_open_generic,
1050	.read		= ftrace_profile_read,
1051	.write		= ftrace_profile_write,
1052	.llseek		= default_llseek,
1053};
1054
1055/* used to initialize the real stat files */
1056static struct tracer_stat function_stats __initdata = {
1057	.name		= "functions",
1058	.stat_start	= function_stat_start,
1059	.stat_next	= function_stat_next,
1060	.stat_cmp	= function_stat_cmp,
1061	.stat_headers	= function_stat_headers,
1062	.stat_show	= function_stat_show
1063};
1064
1065static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
1066{
1067	struct ftrace_profile_stat *stat;
1068	struct dentry *entry;
1069	char *name;
1070	int ret;
1071	int cpu;
1072
1073	for_each_possible_cpu(cpu) {
1074		stat = &per_cpu(ftrace_profile_stats, cpu);
1075
1076		/* allocate enough for function name + cpu number */
1077		name = kmalloc(32, GFP_KERNEL);
1078		if (!name) {
1079			/*
1080			 * The files created are permanent, if something happens
1081			 * we still do not free memory.
1082			 */
1083			WARN(1,
1084			     "Could not allocate stat file for cpu %d\n",
1085			     cpu);
1086			return;
1087		}
1088		stat->stat = function_stats;
1089		snprintf(name, 32, "function%d", cpu);
1090		stat->stat.name = name;
1091		ret = register_stat_tracer(&stat->stat);
1092		if (ret) {
1093			WARN(1,
1094			     "Could not register function stat for cpu %d\n",
1095			     cpu);
1096			kfree(name);
1097			return;
1098		}
1099	}
1100
1101	entry = debugfs_create_file("function_profile_enabled", 0644,
1102				    d_tracer, NULL, &ftrace_profile_fops);
1103	if (!entry)
1104		pr_warning("Could not create debugfs "
1105			   "'function_profile_enabled' entry\n");
1106}
1107
1108#else /* CONFIG_FUNCTION_PROFILER */
1109static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
1110{
1111}
1112#endif /* CONFIG_FUNCTION_PROFILER */
1113
1114static struct pid * const ftrace_swapper_pid = &init_struct_pid;
1115
1116#ifdef CONFIG_DYNAMIC_FTRACE
1117
 
 
 
 
 
 
 
 
1118#ifndef CONFIG_FTRACE_MCOUNT_RECORD
1119# error Dynamic ftrace depends on MCOUNT_RECORD
1120#endif
1121
1122static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
1123
1124struct ftrace_func_probe {
1125	struct hlist_node	node;
1126	struct ftrace_probe_ops	*ops;
1127	unsigned long		flags;
1128	unsigned long		ip;
1129	void			*data;
1130	struct list_head	free_list;
1131};
1132
1133struct ftrace_func_entry {
1134	struct hlist_node hlist;
1135	unsigned long ip;
1136};
1137
1138struct ftrace_hash {
1139	unsigned long		size_bits;
1140	struct hlist_head	*buckets;
1141	unsigned long		count;
1142	struct rcu_head		rcu;
1143};
1144
1145/*
1146 * We make these constant because no one should touch them,
1147 * but they are used as the default "empty hash", to avoid allocating
1148 * it all the time. These are in a read only section such that if
1149 * anyone does try to modify it, it will cause an exception.
1150 */
1151static const struct hlist_head empty_buckets[1];
1152static const struct ftrace_hash empty_hash = {
1153	.buckets = (struct hlist_head *)empty_buckets,
1154};
1155#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1156
1157static struct ftrace_ops global_ops = {
1158	.func			= ftrace_stub,
1159	.notrace_hash		= EMPTY_HASH,
1160	.filter_hash		= EMPTY_HASH,
1161	.flags			= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
1162	INIT_REGEX_LOCK(global_ops)
 
1163};
1164
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1165struct ftrace_page {
1166	struct ftrace_page	*next;
1167	struct dyn_ftrace	*records;
1168	int			index;
1169	int			size;
1170};
1171
1172#define ENTRY_SIZE sizeof(struct dyn_ftrace)
1173#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1174
1175/* estimate from running different kernels */
1176#define NR_TO_INIT		10000
1177
1178static struct ftrace_page	*ftrace_pages_start;
1179static struct ftrace_page	*ftrace_pages;
1180
1181static bool ftrace_hash_empty(struct ftrace_hash *hash)
 
1182{
1183	return !hash || !hash->count;
 
 
 
1184}
1185
1186static struct ftrace_func_entry *
1187ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
 
1188{
1189	unsigned long key;
1190	struct ftrace_func_entry *entry;
1191	struct hlist_head *hhd;
1192
1193	if (ftrace_hash_empty(hash))
1194		return NULL;
1195
1196	if (hash->size_bits > 0)
1197		key = hash_long(ip, hash->size_bits);
1198	else
1199		key = 0;
1200
1201	hhd = &hash->buckets[key];
1202
1203	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1204		if (entry->ip == ip)
1205			return entry;
1206	}
1207	return NULL;
1208}
1209
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1210static void __add_hash_entry(struct ftrace_hash *hash,
1211			     struct ftrace_func_entry *entry)
1212{
1213	struct hlist_head *hhd;
1214	unsigned long key;
1215
1216	if (hash->size_bits)
1217		key = hash_long(entry->ip, hash->size_bits);
1218	else
1219		key = 0;
1220
1221	hhd = &hash->buckets[key];
1222	hlist_add_head(&entry->hlist, hhd);
1223	hash->count++;
1224}
1225
1226static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1227{
1228	struct ftrace_func_entry *entry;
1229
1230	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1231	if (!entry)
1232		return -ENOMEM;
1233
1234	entry->ip = ip;
1235	__add_hash_entry(hash, entry);
1236
1237	return 0;
1238}
1239
1240static void
1241free_hash_entry(struct ftrace_hash *hash,
1242		  struct ftrace_func_entry *entry)
1243{
1244	hlist_del(&entry->hlist);
1245	kfree(entry);
1246	hash->count--;
1247}
1248
1249static void
1250remove_hash_entry(struct ftrace_hash *hash,
1251		  struct ftrace_func_entry *entry)
1252{
1253	hlist_del(&entry->hlist);
1254	hash->count--;
1255}
1256
1257static void ftrace_hash_clear(struct ftrace_hash *hash)
1258{
1259	struct hlist_head *hhd;
1260	struct hlist_node *tn;
1261	struct ftrace_func_entry *entry;
1262	int size = 1 << hash->size_bits;
1263	int i;
1264
1265	if (!hash->count)
1266		return;
1267
1268	for (i = 0; i < size; i++) {
1269		hhd = &hash->buckets[i];
1270		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1271			free_hash_entry(hash, entry);
1272	}
1273	FTRACE_WARN_ON(hash->count);
1274}
1275
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1276static void free_ftrace_hash(struct ftrace_hash *hash)
1277{
1278	if (!hash || hash == EMPTY_HASH)
1279		return;
1280	ftrace_hash_clear(hash);
1281	kfree(hash->buckets);
1282	kfree(hash);
1283}
1284
1285static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1286{
1287	struct ftrace_hash *hash;
1288
1289	hash = container_of(rcu, struct ftrace_hash, rcu);
1290	free_ftrace_hash(hash);
1291}
1292
1293static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1294{
1295	if (!hash || hash == EMPTY_HASH)
1296		return;
1297	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1298}
1299
 
 
 
 
1300void ftrace_free_filter(struct ftrace_ops *ops)
1301{
1302	ftrace_ops_init(ops);
1303	free_ftrace_hash(ops->filter_hash);
1304	free_ftrace_hash(ops->notrace_hash);
1305}
 
1306
1307static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1308{
1309	struct ftrace_hash *hash;
1310	int size;
1311
1312	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1313	if (!hash)
1314		return NULL;
1315
1316	size = 1 << size_bits;
1317	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1318
1319	if (!hash->buckets) {
1320		kfree(hash);
1321		return NULL;
1322	}
1323
1324	hash->size_bits = size_bits;
1325
1326	return hash;
1327}
1328
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1329static struct ftrace_hash *
1330alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1331{
1332	struct ftrace_func_entry *entry;
1333	struct ftrace_hash *new_hash;
1334	int size;
1335	int ret;
1336	int i;
1337
1338	new_hash = alloc_ftrace_hash(size_bits);
1339	if (!new_hash)
1340		return NULL;
1341
 
 
 
1342	/* Empty hash? */
1343	if (ftrace_hash_empty(hash))
1344		return new_hash;
1345
1346	size = 1 << hash->size_bits;
1347	for (i = 0; i < size; i++) {
1348		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1349			ret = add_hash_entry(new_hash, entry->ip);
1350			if (ret < 0)
1351				goto free_hash;
1352		}
1353	}
1354
1355	FTRACE_WARN_ON(new_hash->count != hash->count);
1356
1357	return new_hash;
1358
1359 free_hash:
1360	free_ftrace_hash(new_hash);
1361	return NULL;
1362}
1363
1364static void
1365ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
1366static void
1367ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
1368
1369static int
1370ftrace_hash_move(struct ftrace_ops *ops, int enable,
1371		 struct ftrace_hash **dst, struct ftrace_hash *src)
 
1372{
1373	struct ftrace_func_entry *entry;
1374	struct hlist_node *tn;
1375	struct hlist_head *hhd;
1376	struct ftrace_hash *old_hash;
1377	struct ftrace_hash *new_hash;
1378	int size = src->count;
 
1379	int bits = 0;
1380	int ret;
1381	int i;
1382
1383	/*
1384	 * Remove the current set, update the hash and add
1385	 * them back.
1386	 */
1387	ftrace_hash_rec_disable(ops, enable);
1388
1389	/*
1390	 * If the new source is empty, just free dst and assign it
1391	 * the empty_hash.
1392	 */
1393	if (!src->count) {
1394		free_ftrace_hash_rcu(*dst);
1395		rcu_assign_pointer(*dst, EMPTY_HASH);
1396		/* still need to update the function records */
1397		ret = 0;
1398		goto out;
1399	}
1400
1401	/*
1402	 * Make the hash size about 1/2 the # found
1403	 */
1404	for (size /= 2; size; size >>= 1)
1405		bits++;
1406
1407	/* Don't allocate too much */
1408	if (bits > FTRACE_HASH_MAX_BITS)
1409		bits = FTRACE_HASH_MAX_BITS;
1410
1411	ret = -ENOMEM;
1412	new_hash = alloc_ftrace_hash(bits);
1413	if (!new_hash)
1414		goto out;
 
 
1415
1416	size = 1 << src->size_bits;
1417	for (i = 0; i < size; i++) {
1418		hhd = &src->buckets[i];
1419		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1420			remove_hash_entry(src, entry);
1421			__add_hash_entry(new_hash, entry);
1422		}
1423	}
 
 
1424
1425	old_hash = *dst;
1426	rcu_assign_pointer(*dst, new_hash);
1427	free_ftrace_hash_rcu(old_hash);
 
1428
1429	ret = 0;
1430 out:
1431	/*
1432	 * Enable regardless of ret:
1433	 *  On success, we enable the new hash.
1434	 *  On failure, we re-enable the original hash.
1435	 */
1436	ftrace_hash_rec_enable(ops, enable);
 
1437
1438	return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1439}
1440
1441/*
1442 * Test the hashes for this ops to see if we want to call
1443 * the ops->func or not.
1444 *
1445 * It's a match if the ip is in the ops->filter_hash or
1446 * the filter_hash does not exist or is empty,
1447 *  AND
1448 * the ip is not in the ops->notrace_hash.
1449 *
1450 * This needs to be called with preemption disabled as
1451 * the hashes are freed with call_rcu_sched().
1452 */
1453static int
1454ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1455{
1456	struct ftrace_hash *filter_hash;
1457	struct ftrace_hash *notrace_hash;
1458	int ret;
1459
1460#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1461	/*
1462	 * There's a small race when adding ops that the ftrace handler
1463	 * that wants regs, may be called without them. We can not
1464	 * allow that handler to be called if regs is NULL.
1465	 */
1466	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1467		return 0;
1468#endif
1469
1470	filter_hash = rcu_dereference_raw_notrace(ops->filter_hash);
1471	notrace_hash = rcu_dereference_raw_notrace(ops->notrace_hash);
1472
1473	if ((ftrace_hash_empty(filter_hash) ||
1474	     ftrace_lookup_ip(filter_hash, ip)) &&
1475	    (ftrace_hash_empty(notrace_hash) ||
1476	     !ftrace_lookup_ip(notrace_hash, ip)))
1477		ret = 1;
1478	else
1479		ret = 0;
1480
1481	return ret;
1482}
1483
1484/*
1485 * This is a double for. Do not use 'break' to break out of the loop,
1486 * you must use a goto.
1487 */
1488#define do_for_each_ftrace_rec(pg, rec)					\
1489	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1490		int _____i;						\
1491		for (_____i = 0; _____i < pg->index; _____i++) {	\
1492			rec = &pg->records[_____i];
1493
1494#define while_for_each_ftrace_rec()		\
1495		}				\
1496	}
1497
1498
1499static int ftrace_cmp_recs(const void *a, const void *b)
1500{
1501	const struct dyn_ftrace *key = a;
1502	const struct dyn_ftrace *rec = b;
1503
1504	if (key->flags < rec->ip)
1505		return -1;
1506	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1507		return 1;
1508	return 0;
1509}
1510
1511static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1512{
1513	struct ftrace_page *pg;
1514	struct dyn_ftrace *rec;
1515	struct dyn_ftrace key;
1516
1517	key.ip = start;
1518	key.flags = end;	/* overload flags, as it is unsigned long */
1519
1520	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1521		if (end < pg->records[0].ip ||
1522		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1523			continue;
1524		rec = bsearch(&key, pg->records, pg->index,
1525			      sizeof(struct dyn_ftrace),
1526			      ftrace_cmp_recs);
1527		if (rec)
1528			return rec->ip;
1529	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1530
1531	return 0;
1532}
1533
1534/**
1535 * ftrace_location - return true if the ip giving is a traced location
1536 * @ip: the instruction pointer to check
1537 *
1538 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1539 * That is, the instruction that is either a NOP or call to
1540 * the function tracer. It checks the ftrace internal tables to
1541 * determine if the address belongs or not.
1542 */
1543unsigned long ftrace_location(unsigned long ip)
1544{
1545	return ftrace_location_range(ip, ip);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1546}
1547
1548/**
1549 * ftrace_text_reserved - return true if range contains an ftrace location
1550 * @start: start of range to search
1551 * @end: end of range to search (inclusive). @end points to the last byte to check.
1552 *
1553 * Returns 1 if @start and @end contains a ftrace location.
1554 * That is, the instruction that is either a NOP or call to
1555 * the function tracer. It checks the ftrace internal tables to
1556 * determine if the address belongs or not.
1557 */
1558int ftrace_text_reserved(const void *start, const void *end)
1559{
1560	unsigned long ret;
1561
1562	ret = ftrace_location_range((unsigned long)start,
1563				    (unsigned long)end);
1564
1565	return (int)!!ret;
1566}
1567
1568static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1569				     int filter_hash,
1570				     bool inc)
1571{
1572	struct ftrace_hash *hash;
1573	struct ftrace_hash *other_hash;
1574	struct ftrace_page *pg;
1575	struct dyn_ftrace *rec;
 
1576	int count = 0;
1577	int all = 0;
1578
1579	/* Only update if the ops has been registered */
1580	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1581		return;
1582
1583	/*
1584	 * In the filter_hash case:
1585	 *   If the count is zero, we update all records.
1586	 *   Otherwise we just update the items in the hash.
1587	 *
1588	 * In the notrace_hash case:
1589	 *   We enable the update in the hash.
1590	 *   As disabling notrace means enabling the tracing,
1591	 *   and enabling notrace means disabling, the inc variable
1592	 *   gets inversed.
1593	 */
1594	if (filter_hash) {
1595		hash = ops->filter_hash;
1596		other_hash = ops->notrace_hash;
1597		if (ftrace_hash_empty(hash))
1598			all = 1;
1599	} else {
1600		inc = !inc;
1601		hash = ops->notrace_hash;
1602		other_hash = ops->filter_hash;
1603		/*
1604		 * If the notrace hash has no items,
1605		 * then there's nothing to do.
1606		 */
1607		if (ftrace_hash_empty(hash))
1608			return;
1609	}
1610
1611	do_for_each_ftrace_rec(pg, rec) {
1612		int in_other_hash = 0;
1613		int in_hash = 0;
1614		int match = 0;
1615
 
 
 
1616		if (all) {
1617			/*
1618			 * Only the filter_hash affects all records.
1619			 * Update if the record is not in the notrace hash.
1620			 */
1621			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1622				match = 1;
1623		} else {
1624			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1625			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1626
1627			/*
 
 
1628			 *
 
 
 
 
 
1629			 */
1630			if (filter_hash && in_hash && !in_other_hash)
1631				match = 1;
1632			else if (!filter_hash && in_hash &&
1633				 (in_other_hash || ftrace_hash_empty(other_hash)))
1634				match = 1;
1635		}
1636		if (!match)
1637			continue;
1638
1639		if (inc) {
1640			rec->flags++;
1641			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1642				return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1643			/*
1644			 * If any ops wants regs saved for this function
1645			 * then all ops will get saved regs.
1646			 */
1647			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1648				rec->flags |= FTRACE_FL_REGS;
1649		} else {
1650			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1651				return;
1652			rec->flags--;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1653		}
1654		count++;
 
 
 
 
1655		/* Shortcut, if we handled all records, we are done. */
1656		if (!all && count == hash->count)
1657			return;
1658	} while_for_each_ftrace_rec();
 
 
1659}
1660
1661static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1662				    int filter_hash)
1663{
1664	__ftrace_hash_rec_update(ops, filter_hash, 0);
1665}
1666
1667static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1668				   int filter_hash)
1669{
1670	__ftrace_hash_rec_update(ops, filter_hash, 1);
1671}
1672
1673static void print_ip_ins(const char *fmt, unsigned char *p)
 
1674{
1675	int i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1676
1677	printk(KERN_CONT "%s", fmt);
 
 
 
 
 
1678
1679	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1680		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1681}
1682
1683/**
1684 * ftrace_bug - report and shutdown function tracer
1685 * @failed: The failed type (EFAULT, EINVAL, EPERM)
1686 * @ip: The address that failed
1687 *
1688 * The arch code that enables or disables the function tracing
1689 * can call ftrace_bug() when it has detected a problem in
1690 * modifying the code. @failed should be one of either:
1691 * EFAULT - if the problem happens on reading the @ip address
1692 * EINVAL - if what is read at @ip is not what was expected
1693 * EPERM - if the problem happens on writting to the @ip address
1694 */
1695void ftrace_bug(int failed, unsigned long ip)
1696{
 
 
 
 
1697	switch (failed) {
1698	case -EFAULT:
1699		FTRACE_WARN_ON_ONCE(1);
1700		pr_info("ftrace faulted on modifying ");
1701		print_ip_sym(ip);
1702		break;
1703	case -EINVAL:
1704		FTRACE_WARN_ON_ONCE(1);
1705		pr_info("ftrace failed to modify ");
1706		print_ip_sym(ip);
1707		print_ip_ins(" actual: ", (unsigned char *)ip);
1708		printk(KERN_CONT "\n");
 
 
 
 
1709		break;
1710	case -EPERM:
1711		FTRACE_WARN_ON_ONCE(1);
1712		pr_info("ftrace faulted on writing ");
1713		print_ip_sym(ip);
1714		break;
1715	default:
1716		FTRACE_WARN_ON_ONCE(1);
1717		pr_info("ftrace faulted on unknown error ");
1718		print_ip_sym(ip);
1719	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1720}
1721
1722static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
1723{
1724	unsigned long flag = 0UL;
1725
 
 
 
 
 
1726	/*
1727	 * If we are updating calls:
1728	 *
1729	 *   If the record has a ref count, then we need to enable it
1730	 *   because someone is using it.
1731	 *
1732	 *   Otherwise we make sure its disabled.
1733	 *
1734	 * If we are disabling calls, then disable all records that
1735	 * are enabled.
1736	 */
1737	if (enable && (rec->flags & ~FTRACE_FL_MASK))
1738		flag = FTRACE_FL_ENABLED;
1739
1740	/*
1741	 * If enabling and the REGS flag does not match the REGS_EN, then
1742	 * do not ignore this record. Set flags to fail the compare against
1743	 * ENABLED.
1744	 */
1745	if (flag &&
1746	    (!(rec->flags & FTRACE_FL_REGS) != !(rec->flags & FTRACE_FL_REGS_EN)))
1747		flag |= FTRACE_FL_REGS;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1748
1749	/* If the state of this record hasn't changed, then do nothing */
1750	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1751		return FTRACE_UPDATE_IGNORE;
1752
1753	if (flag) {
1754		/* Save off if rec is being enabled (for return value) */
1755		flag ^= rec->flags & FTRACE_FL_ENABLED;
1756
1757		if (update) {
1758			rec->flags |= FTRACE_FL_ENABLED;
1759			if (flag & FTRACE_FL_REGS) {
1760				if (rec->flags & FTRACE_FL_REGS)
1761					rec->flags |= FTRACE_FL_REGS_EN;
1762				else
1763					rec->flags &= ~FTRACE_FL_REGS_EN;
1764			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1765		}
1766
1767		/*
1768		 * If this record is being updated from a nop, then
1769		 *   return UPDATE_MAKE_CALL.
1770		 * Otherwise, if the EN flag is set, then return
1771		 *   UPDATE_MODIFY_CALL_REGS to tell the caller to convert
1772		 *   from the non-save regs, to a save regs function.
1773		 * Otherwise,
1774		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
1775		 *   from the save regs, to a non-save regs function.
 
1776		 */
1777		if (flag & FTRACE_FL_ENABLED)
 
1778			return FTRACE_UPDATE_MAKE_CALL;
1779		else if (rec->flags & FTRACE_FL_REGS_EN)
1780			return FTRACE_UPDATE_MODIFY_CALL_REGS;
1781		else
1782			return FTRACE_UPDATE_MODIFY_CALL;
1783	}
1784
1785	if (update) {
1786		/* If there's no more users, clear all flags */
1787		if (!(rec->flags & ~FTRACE_FL_MASK))
1788			rec->flags = 0;
1789		else
1790			/* Just disable the record (keep REGS state) */
1791			rec->flags &= ~FTRACE_FL_ENABLED;
 
 
 
 
1792	}
1793
 
1794	return FTRACE_UPDATE_MAKE_NOP;
1795}
1796
1797/**
1798 * ftrace_update_record, set a record that now is tracing or not
1799 * @rec: the record to update
1800 * @enable: set to 1 if the record is tracing, zero to force disable
1801 *
1802 * The records that represent all functions that can be traced need
1803 * to be updated when tracing has been enabled.
1804 */
1805int ftrace_update_record(struct dyn_ftrace *rec, int enable)
1806{
1807	return ftrace_check_record(rec, enable, 1);
1808}
1809
1810/**
1811 * ftrace_test_record, check if the record has been enabled or not
1812 * @rec: the record to test
1813 * @enable: set to 1 to check if enabled, 0 if it is disabled
1814 *
1815 * The arch code may need to test if a record is already set to
1816 * tracing to determine how to modify the function code that it
1817 * represents.
1818 */
1819int ftrace_test_record(struct dyn_ftrace *rec, int enable)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1820{
1821	return ftrace_check_record(rec, enable, 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1822}
1823
1824static int
1825__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1826{
1827	unsigned long ftrace_old_addr;
1828	unsigned long ftrace_addr;
1829	int ret;
1830
 
 
 
 
 
1831	ret = ftrace_update_record(rec, enable);
1832
1833	if (rec->flags & FTRACE_FL_REGS)
1834		ftrace_addr = (unsigned long)FTRACE_REGS_ADDR;
1835	else
1836		ftrace_addr = (unsigned long)FTRACE_ADDR;
1837
1838	switch (ret) {
1839	case FTRACE_UPDATE_IGNORE:
1840		return 0;
1841
1842	case FTRACE_UPDATE_MAKE_CALL:
 
1843		return ftrace_make_call(rec, ftrace_addr);
1844
1845	case FTRACE_UPDATE_MAKE_NOP:
1846		return ftrace_make_nop(NULL, rec, ftrace_addr);
 
1847
1848	case FTRACE_UPDATE_MODIFY_CALL_REGS:
1849	case FTRACE_UPDATE_MODIFY_CALL:
1850		if (rec->flags & FTRACE_FL_REGS)
1851			ftrace_old_addr = (unsigned long)FTRACE_ADDR;
1852		else
1853			ftrace_old_addr = (unsigned long)FTRACE_REGS_ADDR;
1854
1855		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
1856	}
1857
1858	return -1; /* unknow ftrace bug */
1859}
1860
1861void __weak ftrace_replace_code(int enable)
1862{
1863	struct dyn_ftrace *rec;
1864	struct ftrace_page *pg;
 
 
1865	int failed;
1866
1867	if (unlikely(ftrace_disabled))
1868		return;
1869
1870	do_for_each_ftrace_rec(pg, rec) {
 
 
 
 
1871		failed = __ftrace_replace_code(rec, enable);
1872		if (failed) {
1873			ftrace_bug(failed, rec->ip);
1874			/* Stop processing */
1875			return;
1876		}
 
 
1877	} while_for_each_ftrace_rec();
1878}
1879
1880struct ftrace_rec_iter {
1881	struct ftrace_page	*pg;
1882	int			index;
1883};
1884
1885/**
1886 * ftrace_rec_iter_start, start up iterating over traced functions
1887 *
1888 * Returns an iterator handle that is used to iterate over all
1889 * the records that represent address locations where functions
1890 * are traced.
1891 *
1892 * May return NULL if no records are available.
1893 */
1894struct ftrace_rec_iter *ftrace_rec_iter_start(void)
1895{
1896	/*
1897	 * We only use a single iterator.
1898	 * Protected by the ftrace_lock mutex.
1899	 */
1900	static struct ftrace_rec_iter ftrace_rec_iter;
1901	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
1902
1903	iter->pg = ftrace_pages_start;
1904	iter->index = 0;
1905
1906	/* Could have empty pages */
1907	while (iter->pg && !iter->pg->index)
1908		iter->pg = iter->pg->next;
1909
1910	if (!iter->pg)
1911		return NULL;
1912
1913	return iter;
1914}
1915
1916/**
1917 * ftrace_rec_iter_next, get the next record to process.
1918 * @iter: The handle to the iterator.
1919 *
1920 * Returns the next iterator after the given iterator @iter.
1921 */
1922struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
1923{
1924	iter->index++;
1925
1926	if (iter->index >= iter->pg->index) {
1927		iter->pg = iter->pg->next;
1928		iter->index = 0;
1929
1930		/* Could have empty pages */
1931		while (iter->pg && !iter->pg->index)
1932			iter->pg = iter->pg->next;
1933	}
1934
1935	if (!iter->pg)
1936		return NULL;
1937
1938	return iter;
1939}
1940
1941/**
1942 * ftrace_rec_iter_record, get the record at the iterator location
1943 * @iter: The current iterator location
1944 *
1945 * Returns the record that the current @iter is at.
1946 */
1947struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
1948{
1949	return &iter->pg->records[iter->index];
1950}
1951
1952static int
1953ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1954{
1955	unsigned long ip;
1956	int ret;
1957
1958	ip = rec->ip;
1959
1960	if (unlikely(ftrace_disabled))
1961		return 0;
1962
1963	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1964	if (ret) {
1965		ftrace_bug(ret, ip);
 
1966		return 0;
1967	}
1968	return 1;
1969}
1970
1971/*
1972 * archs can override this function if they must do something
1973 * before the modifying code is performed.
1974 */
1975int __weak ftrace_arch_code_modify_prepare(void)
1976{
1977	return 0;
1978}
1979
1980/*
1981 * archs can override this function if they must do something
1982 * after the modifying code is performed.
1983 */
1984int __weak ftrace_arch_code_modify_post_process(void)
1985{
1986	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
1987}
1988
1989void ftrace_modify_all_code(int command)
1990{
1991	int update = command & FTRACE_UPDATE_TRACE_FUNC;
 
1992	int err = 0;
1993
 
 
 
1994	/*
1995	 * If the ftrace_caller calls a ftrace_ops func directly,
1996	 * we need to make sure that it only traces functions it
1997	 * expects to trace. When doing the switch of functions,
1998	 * we need to update to the ftrace_ops_list_func first
1999	 * before the transition between old and new calls are set,
2000	 * as the ftrace_ops_list_func will check the ops hashes
2001	 * to make sure the ops are having the right functions
2002	 * traced.
2003	 */
2004	if (update) {
2005		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
2006		if (FTRACE_WARN_ON(err))
2007			return;
2008	}
2009
2010	if (command & FTRACE_UPDATE_CALLS)
2011		ftrace_replace_code(1);
2012	else if (command & FTRACE_DISABLE_CALLS)
2013		ftrace_replace_code(0);
2014
2015	if (update && ftrace_trace_function != ftrace_ops_list_func) {
2016		function_trace_op = set_function_trace_op;
2017		smp_wmb();
2018		/* If irqs are disabled, we are in stop machine */
2019		if (!irqs_disabled())
2020			smp_call_function(ftrace_sync_ipi, NULL, 1);
2021		err = ftrace_update_ftrace_func(ftrace_trace_function);
2022		if (FTRACE_WARN_ON(err))
2023			return;
2024	}
2025
2026	if (command & FTRACE_START_FUNC_RET)
2027		err = ftrace_enable_ftrace_graph_caller();
2028	else if (command & FTRACE_STOP_FUNC_RET)
2029		err = ftrace_disable_ftrace_graph_caller();
2030	FTRACE_WARN_ON(err);
2031}
2032
2033static int __ftrace_modify_code(void *data)
2034{
2035	int *command = data;
2036
2037	ftrace_modify_all_code(*command);
2038
2039	return 0;
2040}
2041
2042/**
2043 * ftrace_run_stop_machine, go back to the stop machine method
2044 * @command: The command to tell ftrace what to do
2045 *
2046 * If an arch needs to fall back to the stop machine method, the
2047 * it can call this function.
2048 */
2049void ftrace_run_stop_machine(int command)
2050{
2051	stop_machine(__ftrace_modify_code, &command, NULL);
2052}
2053
2054/**
2055 * arch_ftrace_update_code, modify the code to trace or not trace
2056 * @command: The command that needs to be done
2057 *
2058 * Archs can override this function if it does not need to
2059 * run stop_machine() to modify code.
2060 */
2061void __weak arch_ftrace_update_code(int command)
2062{
2063	ftrace_run_stop_machine(command);
2064}
2065
2066static void ftrace_run_update_code(int command)
2067{
2068	int ret;
2069
2070	ret = ftrace_arch_code_modify_prepare();
2071	FTRACE_WARN_ON(ret);
2072	if (ret)
2073		return;
2074	/*
2075	 * Do not call function tracer while we update the code.
2076	 * We are in stop machine.
2077	 */
2078	function_trace_stop++;
2079
2080	/*
2081	 * By default we use stop_machine() to modify the code.
2082	 * But archs can do what ever they want as long as it
2083	 * is safe. The stop_machine() is the safest, but also
2084	 * produces the most overhead.
2085	 */
2086	arch_ftrace_update_code(command);
2087
2088	function_trace_stop--;
 
2089
2090	ret = ftrace_arch_code_modify_post_process();
2091	FTRACE_WARN_ON(ret);
 
 
 
 
 
 
 
 
2092}
2093
2094static ftrace_func_t saved_ftrace_func;
2095static int ftrace_start_up;
2096static int global_start_up;
2097
2098static void control_ops_free(struct ftrace_ops *ops)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2099{
2100	free_percpu(ops->disabled);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2101}
2102
2103static void ftrace_startup_enable(int command)
2104{
2105	if (saved_ftrace_func != ftrace_trace_function) {
2106		saved_ftrace_func = ftrace_trace_function;
2107		command |= FTRACE_UPDATE_TRACE_FUNC;
2108	}
2109
2110	if (!command || !ftrace_enabled)
2111		return;
2112
2113	ftrace_run_update_code(command);
2114}
2115
2116static int ftrace_startup(struct ftrace_ops *ops, int command)
 
 
 
 
 
 
 
2117{
2118	bool hash_enable = true;
2119	int ret;
2120
2121	if (unlikely(ftrace_disabled))
2122		return -ENODEV;
2123
2124	ret = __register_ftrace_function(ops);
2125	if (ret)
2126		return ret;
2127
2128	ftrace_start_up++;
2129	command |= FTRACE_UPDATE_CALLS;
2130
2131	/* ops marked global share the filter hashes */
2132	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
2133		ops = &global_ops;
2134		/* Don't update hash if global is already set */
2135		if (global_start_up)
2136			hash_enable = false;
2137		global_start_up++;
 
 
 
 
 
 
 
 
 
 
 
 
2138	}
2139
2140	ops->flags |= FTRACE_OPS_FL_ENABLED;
2141	if (hash_enable)
2142		ftrace_hash_rec_enable(ops, 1);
2143
2144	ftrace_startup_enable(command);
2145
 
 
 
 
 
 
 
 
 
 
 
 
2146	return 0;
2147}
2148
2149static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2150{
2151	bool hash_disable = true;
2152	int ret;
2153
2154	if (unlikely(ftrace_disabled))
2155		return -ENODEV;
2156
2157	ret = __unregister_ftrace_function(ops);
2158	if (ret)
2159		return ret;
2160
2161	ftrace_start_up--;
2162	/*
2163	 * Just warn in case of unbalance, no need to kill ftrace, it's not
2164	 * critical but the ftrace_call callers may be never nopped again after
2165	 * further ftrace uses.
2166	 */
2167	WARN_ON_ONCE(ftrace_start_up < 0);
2168
2169	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
2170		ops = &global_ops;
2171		global_start_up--;
2172		WARN_ON_ONCE(global_start_up < 0);
2173		/* Don't update hash if global still has users */
2174		if (global_start_up) {
2175			WARN_ON_ONCE(!ftrace_start_up);
2176			hash_disable = false;
2177		}
2178	}
2179
2180	if (hash_disable)
2181		ftrace_hash_rec_disable(ops, 1);
2182
2183	if (ops != &global_ops || !global_start_up)
2184		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2185
2186	command |= FTRACE_UPDATE_CALLS;
2187
2188	if (saved_ftrace_func != ftrace_trace_function) {
2189		saved_ftrace_func = ftrace_trace_function;
2190		command |= FTRACE_UPDATE_TRACE_FUNC;
2191	}
2192
2193	if (!command || !ftrace_enabled) {
2194		/*
2195		 * If these are control ops, they still need their
2196		 * per_cpu field freed. Since, function tracing is
2197		 * not currently active, we can just free them
2198		 * without synchronizing all CPUs.
2199		 */
2200		if (ops->flags & FTRACE_OPS_FL_CONTROL)
2201			control_ops_free(ops);
2202		return 0;
2203	}
 
 
2204
2205	ftrace_run_update_code(command);
2206
2207	/*
2208	 * Dynamic ops may be freed, we must make sure that all
2209	 * callers are done before leaving this function.
2210	 * The same goes for freeing the per_cpu data of the control
2211	 * ops.
2212	 *
2213	 * Again, normal synchronize_sched() is not good enough.
2214	 * We need to do a hard force of sched synchronization.
2215	 * This is because we use preempt_disable() to do RCU, but
2216	 * the function tracers can be called where RCU is not watching
2217	 * (like before user_exit()). We can not rely on the RCU
2218	 * infrastructure to do the synchronization, thus we must do it
2219	 * ourselves.
2220	 */
2221	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_CONTROL)) {
2222		schedule_on_each_cpu(ftrace_sync);
2223
2224		if (ops->flags & FTRACE_OPS_FL_CONTROL)
2225			control_ops_free(ops);
 
 
 
 
 
2226	}
2227
2228	return 0;
2229}
2230
2231static void ftrace_startup_sysctl(void)
2232{
2233	if (unlikely(ftrace_disabled))
2234		return;
2235
2236	/* Force update next time */
2237	saved_ftrace_func = NULL;
2238	/* ftrace_start_up is true if we want ftrace running */
2239	if (ftrace_start_up)
2240		ftrace_run_update_code(FTRACE_UPDATE_CALLS);
2241}
 
 
 
 
 
 
 
 
 
2242
2243static void ftrace_shutdown_sysctl(void)
2244{
2245	if (unlikely(ftrace_disabled))
2246		return;
 
 
 
 
 
2247
2248	/* ftrace_start_up is true if ftrace is running */
2249	if (ftrace_start_up)
2250		ftrace_run_update_code(FTRACE_DISABLE_CALLS);
 
2251}
2252
2253static cycle_t		ftrace_update_time;
2254unsigned long		ftrace_update_tot_cnt;
 
 
2255
2256static inline int ops_traces_mod(struct ftrace_ops *ops)
2257{
2258	/*
2259	 * Filter_hash being empty will default to trace module.
2260	 * But notrace hash requires a test of individual module functions.
2261	 */
2262	return ftrace_hash_empty(ops->filter_hash) &&
2263		ftrace_hash_empty(ops->notrace_hash);
2264}
2265
2266/*
2267 * Check if the current ops references the record.
2268 *
2269 * If the ops traces all functions, then it was already accounted for.
2270 * If the ops does not trace the current record function, skip it.
2271 * If the ops ignores the function via notrace filter, skip it.
2272 */
2273static inline bool
2274ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
2275{
2276	/* If ops isn't enabled, ignore it */
2277	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
2278		return 0;
2279
2280	/* If ops traces all mods, we already accounted for it */
2281	if (ops_traces_mod(ops))
2282		return 0;
2283
2284	/* The function must be in the filter */
2285	if (!ftrace_hash_empty(ops->filter_hash) &&
2286	    !ftrace_lookup_ip(ops->filter_hash, rec->ip))
2287		return 0;
2288
2289	/* If in notrace hash, we ignore it too */
2290	if (ftrace_lookup_ip(ops->notrace_hash, rec->ip))
2291		return 0;
2292
2293	return 1;
2294}
2295
2296static int referenced_filters(struct dyn_ftrace *rec)
2297{
2298	struct ftrace_ops *ops;
2299	int cnt = 0;
2300
2301	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
2302		if (ops_references_rec(ops, rec))
2303		    cnt++;
2304	}
2305
2306	return cnt;
2307}
2308
2309static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2310{
 
2311	struct ftrace_page *pg;
2312	struct dyn_ftrace *p;
2313	cycle_t start, stop;
2314	unsigned long update_cnt = 0;
2315	unsigned long ref = 0;
2316	bool test = false;
2317	int i;
2318
 
 
2319	/*
2320	 * When adding a module, we need to check if tracers are
2321	 * currently enabled and if they are set to trace all functions.
2322	 * If they are, we need to enable the module functions as well
2323	 * as update the reference counts for those function records.
 
 
 
 
 
2324	 */
2325	if (mod) {
2326		struct ftrace_ops *ops;
2327
2328		for (ops = ftrace_ops_list;
2329		     ops != &ftrace_list_end; ops = ops->next) {
2330			if (ops->flags & FTRACE_OPS_FL_ENABLED) {
2331				if (ops_traces_mod(ops))
2332					ref++;
2333				else
2334					test = true;
2335			}
2336		}
2337	}
2338
2339	start = ftrace_now(raw_smp_processor_id());
2340
2341	for (pg = new_pgs; pg; pg = pg->next) {
2342
2343		for (i = 0; i < pg->index; i++) {
2344			int cnt = ref;
2345
2346			/* If something went wrong, bail without enabling anything */
2347			if (unlikely(ftrace_disabled))
2348				return -1;
2349
2350			p = &pg->records[i];
2351			if (test)
2352				cnt += referenced_filters(p);
2353			p->flags = cnt;
2354
2355			/*
2356			 * Do the initial record conversion from mcount jump
2357			 * to the NOP instructions.
2358			 */
2359			if (!ftrace_code_disable(mod, p))
2360				break;
2361
2362			update_cnt++;
2363
2364			/*
2365			 * If the tracing is enabled, go ahead and enable the record.
2366			 *
2367			 * The reason not to enable the record immediatelly is the
2368			 * inherent check of ftrace_make_nop/ftrace_make_call for
2369			 * correct previous instructions.  Making first the NOP
2370			 * conversion puts the module to the correct state, thus
2371			 * passing the ftrace_make_call check.
2372			 */
2373			if (ftrace_start_up && cnt) {
2374				int failed = __ftrace_replace_code(p, 1);
2375				if (failed)
2376					ftrace_bug(failed, p->ip);
2377			}
2378		}
2379	}
2380
2381	stop = ftrace_now(raw_smp_processor_id());
2382	ftrace_update_time = stop - start;
2383	ftrace_update_tot_cnt += update_cnt;
2384
2385	return 0;
2386}
2387
2388static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2389{
2390	int order;
 
2391	int cnt;
2392
2393	if (WARN_ON(!count))
2394		return -EINVAL;
2395
2396	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2397
2398	/*
2399	 * We want to fill as much as possible. No more than a page
2400	 * may be empty.
2401	 */
2402	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
2403		order--;
2404
2405 again:
2406	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2407
2408	if (!pg->records) {
2409		/* if we can't allocate this size, try something smaller */
2410		if (!order)
2411			return -ENOMEM;
2412		order >>= 1;
2413		goto again;
2414	}
2415
 
 
 
2416	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
2417	pg->size = cnt;
2418
2419	if (cnt > count)
2420		cnt = count;
2421
2422	return cnt;
2423}
2424
2425static struct ftrace_page *
2426ftrace_allocate_pages(unsigned long num_to_init)
2427{
2428	struct ftrace_page *start_pg;
2429	struct ftrace_page *pg;
2430	int order;
2431	int cnt;
2432
2433	if (!num_to_init)
2434		return 0;
2435
2436	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
2437	if (!pg)
2438		return NULL;
2439
2440	/*
2441	 * Try to allocate as much as possible in one continues
2442	 * location that fills in all of the space. We want to
2443	 * waste as little space as possible.
2444	 */
2445	for (;;) {
2446		cnt = ftrace_allocate_records(pg, num_to_init);
2447		if (cnt < 0)
2448			goto free_pages;
2449
2450		num_to_init -= cnt;
2451		if (!num_to_init)
2452			break;
2453
2454		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
2455		if (!pg->next)
2456			goto free_pages;
2457
2458		pg = pg->next;
2459	}
2460
2461	return start_pg;
2462
2463 free_pages:
2464	while (start_pg) {
2465		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
2466		free_pages((unsigned long)pg->records, order);
 
 
 
2467		start_pg = pg->next;
2468		kfree(pg);
2469		pg = start_pg;
 
2470	}
2471	pr_info("ftrace: FAILED to allocate memory for functions\n");
2472	return NULL;
2473}
2474
2475#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
2476
2477struct ftrace_iterator {
2478	loff_t				pos;
2479	loff_t				func_pos;
 
2480	struct ftrace_page		*pg;
2481	struct dyn_ftrace		*func;
2482	struct ftrace_func_probe	*probe;
 
2483	struct trace_parser		parser;
2484	struct ftrace_hash		*hash;
2485	struct ftrace_ops		*ops;
2486	int				hidx;
 
 
2487	int				idx;
2488	unsigned			flags;
2489};
2490
2491static void *
2492t_hash_next(struct seq_file *m, loff_t *pos)
2493{
2494	struct ftrace_iterator *iter = m->private;
 
 
 
 
2495	struct hlist_node *hnd = NULL;
2496	struct hlist_head *hhd;
 
2497
2498	(*pos)++;
2499	iter->pos = *pos;
2500
2501	if (iter->probe)
2502		hnd = &iter->probe->node;
2503 retry:
2504	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2505		return NULL;
2506
2507	hhd = &ftrace_func_hash[iter->hidx];
 
 
 
 
 
 
 
 
 
 
 
 
 
2508
2509	if (hlist_empty(hhd)) {
2510		iter->hidx++;
2511		hnd = NULL;
2512		goto retry;
2513	}
2514
2515	if (!hnd)
2516		hnd = hhd->first;
2517	else {
2518		hnd = hnd->next;
2519		if (!hnd) {
2520			iter->hidx++;
2521			goto retry;
2522		}
2523	}
2524
2525	if (WARN_ON_ONCE(!hnd))
2526		return NULL;
2527
2528	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
2529
2530	return iter;
2531}
2532
2533static void *t_hash_start(struct seq_file *m, loff_t *pos)
2534{
2535	struct ftrace_iterator *iter = m->private;
2536	void *p = NULL;
2537	loff_t l;
2538
2539	if (!(iter->flags & FTRACE_ITER_DO_HASH))
2540		return NULL;
2541
2542	if (iter->func_pos > *pos)
2543		return NULL;
2544
2545	iter->hidx = 0;
2546	for (l = 0; l <= (*pos - iter->func_pos); ) {
2547		p = t_hash_next(m, &l);
 
 
2548		if (!p)
2549			break;
2550	}
2551	if (!p)
2552		return NULL;
2553
2554	/* Only set this if we have an item */
2555	iter->flags |= FTRACE_ITER_HASH;
2556
2557	return iter;
2558}
2559
2560static int
2561t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
2562{
2563	struct ftrace_func_probe *rec;
 
 
2564
2565	rec = iter->probe;
2566	if (WARN_ON_ONCE(!rec))
 
 
2567		return -EIO;
2568
2569	if (rec->ops->print)
2570		return rec->ops->print(m, rec->ip, rec->ops, rec->data);
2571
2572	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
 
2573
2574	if (rec->data)
2575		seq_printf(m, ":%p", rec->data);
2576	seq_putc(m, '\n');
2577
2578	return 0;
2579}
2580
2581static void *
2582t_next(struct seq_file *m, void *v, loff_t *pos)
2583{
2584	struct ftrace_iterator *iter = m->private;
2585	struct ftrace_ops *ops = iter->ops;
2586	struct dyn_ftrace *rec = NULL;
2587
2588	if (unlikely(ftrace_disabled))
 
 
 
 
 
 
 
2589		return NULL;
 
2590
2591	if (iter->flags & FTRACE_ITER_HASH)
2592		return t_hash_next(m, pos);
2593
2594	(*pos)++;
2595	iter->pos = iter->func_pos = *pos;
 
 
 
 
 
 
2596
2597	if (iter->flags & FTRACE_ITER_PRINTALL)
2598		return t_hash_start(m, pos);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2599
2600 retry:
2601	if (iter->idx >= iter->pg->index) {
2602		if (iter->pg->next) {
2603			iter->pg = iter->pg->next;
2604			iter->idx = 0;
2605			goto retry;
2606		}
2607	} else {
2608		rec = &iter->pg->records[iter->idx++];
2609		if (((iter->flags & FTRACE_ITER_FILTER) &&
2610		     !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
2611
2612		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
2613		     !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
2614
2615		    ((iter->flags & FTRACE_ITER_ENABLED) &&
2616		     !(rec->flags & FTRACE_FL_ENABLED))) {
2617
2618			rec = NULL;
2619			goto retry;
2620		}
2621	}
2622
2623	if (!rec)
2624		return t_hash_start(m, pos);
2625
 
2626	iter->func = rec;
2627
2628	return iter;
2629}
2630
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2631static void reset_iter_read(struct ftrace_iterator *iter)
2632{
2633	iter->pos = 0;
2634	iter->func_pos = 0;
2635	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
2636}
2637
2638static void *t_start(struct seq_file *m, loff_t *pos)
2639{
2640	struct ftrace_iterator *iter = m->private;
2641	struct ftrace_ops *ops = iter->ops;
2642	void *p = NULL;
2643	loff_t l;
2644
2645	mutex_lock(&ftrace_lock);
2646
2647	if (unlikely(ftrace_disabled))
2648		return NULL;
2649
2650	/*
2651	 * If an lseek was done, then reset and start from beginning.
2652	 */
2653	if (*pos < iter->pos)
2654		reset_iter_read(iter);
2655
2656	/*
2657	 * For set_ftrace_filter reading, if we have the filter
2658	 * off, we can short cut and just print out that all
2659	 * functions are enabled.
2660	 */
2661	if (iter->flags & FTRACE_ITER_FILTER &&
2662	    ftrace_hash_empty(ops->filter_hash)) {
 
2663		if (*pos > 0)
2664			return t_hash_start(m, pos);
2665		iter->flags |= FTRACE_ITER_PRINTALL;
2666		/* reset in case of seek/pread */
2667		iter->flags &= ~FTRACE_ITER_HASH;
2668		return iter;
2669	}
2670
2671	if (iter->flags & FTRACE_ITER_HASH)
2672		return t_hash_start(m, pos);
2673
2674	/*
2675	 * Unfortunately, we need to restart at ftrace_pages_start
2676	 * every time we let go of the ftrace_mutex. This is because
2677	 * those pointers can change without the lock.
2678	 */
2679	iter->pg = ftrace_pages_start;
2680	iter->idx = 0;
2681	for (l = 0; l <= *pos; ) {
2682		p = t_next(m, p, &l);
2683		if (!p)
2684			break;
2685	}
2686
2687	if (!p)
2688		return t_hash_start(m, pos);
2689
2690	return iter;
2691}
2692
2693static void t_stop(struct seq_file *m, void *p)
2694{
2695	mutex_unlock(&ftrace_lock);
2696}
2697
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2698static int t_show(struct seq_file *m, void *v)
2699{
2700	struct ftrace_iterator *iter = m->private;
2701	struct dyn_ftrace *rec;
2702
2703	if (iter->flags & FTRACE_ITER_HASH)
2704		return t_hash_show(m, iter);
 
 
 
2705
2706	if (iter->flags & FTRACE_ITER_PRINTALL) {
2707		seq_printf(m, "#### all functions enabled ####\n");
 
 
 
2708		return 0;
2709	}
2710
2711	rec = iter->func;
2712
2713	if (!rec)
2714		return 0;
2715
2716	seq_printf(m, "%ps", (void *)rec->ip);
2717	if (iter->flags & FTRACE_ITER_ENABLED)
2718		seq_printf(m, " (%ld)%s",
2719			   rec->flags & ~FTRACE_FL_MASK,
2720			   rec->flags & FTRACE_FL_REGS ? " R" : "");
2721	seq_printf(m, "\n");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2722
2723	return 0;
2724}
2725
2726static const struct seq_operations show_ftrace_seq_ops = {
2727	.start = t_start,
2728	.next = t_next,
2729	.stop = t_stop,
2730	.show = t_show,
2731};
2732
2733static int
2734ftrace_avail_open(struct inode *inode, struct file *file)
2735{
2736	struct ftrace_iterator *iter;
 
 
 
 
 
2737
2738	if (unlikely(ftrace_disabled))
2739		return -ENODEV;
2740
2741	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2742	if (iter) {
2743		iter->pg = ftrace_pages_start;
2744		iter->ops = &global_ops;
2745	}
2746
2747	return iter ? 0 : -ENOMEM;
 
 
 
2748}
2749
2750static int
2751ftrace_enabled_open(struct inode *inode, struct file *file)
2752{
2753	struct ftrace_iterator *iter;
2754
2755	if (unlikely(ftrace_disabled))
2756		return -ENODEV;
 
 
 
 
 
 
2757
2758	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2759	if (iter) {
2760		iter->pg = ftrace_pages_start;
2761		iter->flags = FTRACE_ITER_ENABLED;
2762		iter->ops = &global_ops;
2763	}
2764
2765	return iter ? 0 : -ENOMEM;
2766}
 
2767
2768static void ftrace_filter_reset(struct ftrace_hash *hash)
2769{
2770	mutex_lock(&ftrace_lock);
2771	ftrace_hash_clear(hash);
2772	mutex_unlock(&ftrace_lock);
2773}
2774
2775/**
2776 * ftrace_regex_open - initialize function tracer filter files
2777 * @ops: The ftrace_ops that hold the hash filters
2778 * @flag: The type of filter to process
2779 * @inode: The inode, usually passed in to your open routine
2780 * @file: The file, usually passed in to your open routine
2781 *
2782 * ftrace_regex_open() initializes the filter files for the
2783 * @ops. Depending on @flag it may process the filter hash or
2784 * the notrace hash of @ops. With this called from the open
2785 * routine, you can use ftrace_filter_write() for the write
2786 * routine if @flag has FTRACE_ITER_FILTER set, or
2787 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
2788 * tracing_lseek() should be used as the lseek routine, and
2789 * release must call ftrace_regex_release().
2790 */
2791int
2792ftrace_regex_open(struct ftrace_ops *ops, int flag,
2793		  struct inode *inode, struct file *file)
2794{
2795	struct ftrace_iterator *iter;
2796	struct ftrace_hash *hash;
2797	int ret = 0;
 
 
2798
2799	ftrace_ops_init(ops);
2800
2801	if (unlikely(ftrace_disabled))
2802		return -ENODEV;
2803
 
 
 
2804	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2805	if (!iter)
2806		return -ENOMEM;
2807
2808	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2809		kfree(iter);
2810		return -ENOMEM;
2811	}
2812
2813	iter->ops = ops;
2814	iter->flags = flag;
 
2815
2816	mutex_lock(&ops->regex_lock);
2817
2818	if (flag & FTRACE_ITER_NOTRACE)
2819		hash = ops->notrace_hash;
2820	else
2821		hash = ops->filter_hash;
 
 
 
 
 
2822
2823	if (file->f_mode & FMODE_WRITE) {
2824		iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
 
 
 
 
 
 
 
 
2825		if (!iter->hash) {
2826			trace_parser_put(&iter->parser);
2827			kfree(iter);
2828			ret = -ENOMEM;
2829			goto out_unlock;
2830		}
2831	}
 
2832
2833	if ((file->f_mode & FMODE_WRITE) &&
2834	    (file->f_flags & O_TRUNC))
2835		ftrace_filter_reset(iter->hash);
2836
2837	if (file->f_mode & FMODE_READ) {
2838		iter->pg = ftrace_pages_start;
2839
2840		ret = seq_open(file, &show_ftrace_seq_ops);
2841		if (!ret) {
2842			struct seq_file *m = file->private_data;
2843			m->private = iter;
2844		} else {
2845			/* Failed */
2846			free_ftrace_hash(iter->hash);
2847			trace_parser_put(&iter->parser);
2848			kfree(iter);
2849		}
2850	} else
2851		file->private_data = iter;
2852
2853 out_unlock:
2854	mutex_unlock(&ops->regex_lock);
 
 
 
 
 
 
 
2855
2856	return ret;
2857}
2858
2859static int
2860ftrace_filter_open(struct inode *inode, struct file *file)
2861{
2862	struct ftrace_ops *ops = inode->i_private;
2863
 
2864	return ftrace_regex_open(ops,
2865			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
2866			inode, file);
2867}
2868
2869static int
2870ftrace_notrace_open(struct inode *inode, struct file *file)
2871{
2872	struct ftrace_ops *ops = inode->i_private;
2873
 
2874	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
2875				 inode, file);
2876}
2877
2878static int ftrace_match(char *str, char *regex, int len, int type)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2879{
2880	int matched = 0;
2881	int slen;
2882
2883	switch (type) {
 
 
2884	case MATCH_FULL:
2885		if (strcmp(str, regex) == 0)
2886			matched = 1;
2887		break;
2888	case MATCH_FRONT_ONLY:
2889		if (strncmp(str, regex, len) == 0)
2890			matched = 1;
2891		break;
2892	case MATCH_MIDDLE_ONLY:
2893		if (strstr(str, regex))
2894			matched = 1;
2895		break;
2896	case MATCH_END_ONLY:
2897		slen = strlen(str);
2898		if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
 
 
 
 
 
2899			matched = 1;
2900		break;
2901	}
2902
2903	return matched;
2904}
2905
2906static int
2907enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2908{
2909	struct ftrace_func_entry *entry;
2910	int ret = 0;
2911
2912	entry = ftrace_lookup_ip(hash, rec->ip);
2913	if (not) {
2914		/* Do nothing if it doesn't exist */
2915		if (!entry)
2916			return 0;
2917
2918		free_hash_entry(hash, entry);
2919	} else {
2920		/* Do nothing if it exists */
2921		if (entry)
2922			return 0;
2923
2924		ret = add_hash_entry(hash, rec->ip);
2925	}
2926	return ret;
2927}
2928
2929static int
2930ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2931		    char *regex, int len, int type)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2932{
2933	char str[KSYM_SYMBOL_LEN];
2934	char *modname;
2935
2936	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
 
 
 
 
 
 
 
 
2937
2938	if (mod) {
2939		/* module lookup requires matching the module */
2940		if (!modname || strcmp(modname, mod))
 
 
2941			return 0;
 
2942
 
 
 
 
 
 
 
 
 
 
2943		/* blank search means to match all funcs in the mod */
2944		if (!len)
2945			return 1;
2946	}
2947
2948	return ftrace_match(str, regex, len, type);
2949}
2950
2951static int
2952match_records(struct ftrace_hash *hash, char *buff,
2953	      int len, char *mod, int not)
2954{
2955	unsigned search_len = 0;
2956	struct ftrace_page *pg;
2957	struct dyn_ftrace *rec;
2958	int type = MATCH_FULL;
2959	char *search = buff;
 
 
2960	int found = 0;
2961	int ret;
 
2962
2963	if (len) {
2964		type = filter_parse_regex(buff, len, &search, &not);
2965		search_len = strlen(search);
 
 
 
 
 
 
 
2966	}
2967
2968	mutex_lock(&ftrace_lock);
2969
2970	if (unlikely(ftrace_disabled))
2971		goto out_unlock;
2972
 
 
 
 
 
2973	do_for_each_ftrace_rec(pg, rec) {
2974		if (ftrace_match_record(rec, mod, search, search_len, type)) {
2975			ret = enter_record(hash, rec, not);
 
 
 
 
2976			if (ret < 0) {
2977				found = ret;
2978				goto out_unlock;
2979			}
2980			found = 1;
2981		}
 
2982	} while_for_each_ftrace_rec();
2983 out_unlock:
2984	mutex_unlock(&ftrace_lock);
2985
2986	return found;
2987}
2988
2989static int
2990ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2991{
2992	return match_records(hash, buff, len, NULL, 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2993}
2994
2995static int
2996ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
 
 
 
 
 
2997{
2998	int not = 0;
 
 
 
2999
3000	/* blank or '*' mean the same */
3001	if (strcmp(buff, "*") == 0)
3002		buff[0] = 0;
 
 
 
 
 
 
 
 
3003
3004	/* handle the case of 'dont filter this module' */
3005	if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
3006		buff[0] = 0;
3007		not = 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3008	}
 
3009
3010	return match_records(hash, buff, strlen(buff), mod, not);
3011}
 
3012
3013/*
3014 * We register the module command as a template to show others how
3015 * to register the a command as well.
3016 */
3017
3018static int
3019ftrace_mod_callback(struct ftrace_hash *hash,
3020		    char *func, char *cmd, char *param, int enable)
3021{
3022	char *mod;
3023	int ret = -EINVAL;
 
 
 
 
 
3024
3025	/*
3026	 * cmd == 'mod' because we only registered this func
3027	 * for the 'mod' ftrace_func_command.
3028	 * But if you register one func with multiple commands,
3029	 * you can tell which command was used by the cmd
3030	 * parameter.
3031	 */
 
 
3032
3033	/* we must have a module name */
3034	if (!param)
3035		return ret;
3036
3037	mod = strsep(&param, ":");
3038	if (!strlen(mod))
3039		return ret;
3040
3041	ret = ftrace_match_module_records(hash, func, mod);
3042	if (!ret)
3043		ret = -EINVAL;
3044	if (ret < 0)
3045		return ret;
3046
3047	return 0;
3048}
3049
3050static struct ftrace_func_command ftrace_mod_cmd = {
3051	.name			= "mod",
3052	.func			= ftrace_mod_callback,
3053};
3054
3055static int __init ftrace_mod_cmd_init(void)
3056{
3057	return register_ftrace_command(&ftrace_mod_cmd);
3058}
3059core_initcall(ftrace_mod_cmd_init);
3060
3061static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
3062				      struct ftrace_ops *op, struct pt_regs *pt_regs)
3063{
3064	struct ftrace_func_probe *entry;
3065	struct hlist_head *hhd;
3066	unsigned long key;
3067
3068	key = hash_long(ip, FTRACE_HASH_BITS);
3069
3070	hhd = &ftrace_func_hash[key];
3071
3072	if (hlist_empty(hhd))
3073		return;
3074
3075	/*
3076	 * Disable preemption for these calls to prevent a RCU grace
3077	 * period. This syncs the hash iteration and freeing of items
3078	 * on the hash. rcu_read_lock is too dangerous here.
3079	 */
3080	preempt_disable_notrace();
3081	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
3082		if (entry->ip == ip)
3083			entry->ops->func(ip, parent_ip, &entry->data);
3084	}
3085	preempt_enable_notrace();
3086}
3087
3088static struct ftrace_ops trace_probe_ops __read_mostly =
3089{
3090	.func		= function_trace_probe_call,
3091	.flags		= FTRACE_OPS_FL_INITIALIZED,
3092	INIT_REGEX_LOCK(trace_probe_ops)
3093};
3094
3095static int ftrace_probe_registered;
 
 
3096
3097static void __enable_ftrace_function_probe(void)
 
 
 
 
 
3098{
3099	int ret;
3100	int i;
3101
3102	if (ftrace_probe_registered) {
3103		/* still need to update the function call sites */
3104		if (ftrace_enabled)
3105			ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3106		return;
3107	}
 
 
3108
3109	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3110		struct hlist_head *hhd = &ftrace_func_hash[i];
3111		if (hhd->first)
3112			break;
3113	}
3114	/* Nothing registered? */
3115	if (i == FTRACE_FUNC_HASHSIZE)
3116		return;
 
 
 
 
 
 
 
 
3117
3118	ret = ftrace_startup(&trace_probe_ops, 0);
 
 
3119
3120	ftrace_probe_registered = 1;
 
3121}
3122
3123static void __disable_ftrace_function_probe(void)
 
 
 
 
 
 
 
 
 
3124{
3125	int i;
 
 
 
 
 
 
 
 
 
 
 
 
3126
3127	if (!ftrace_probe_registered)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3128		return;
3129
3130	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3131		struct hlist_head *hhd = &ftrace_func_hash[i];
3132		if (hhd->first)
3133			return;
 
 
 
 
 
3134	}
 
 
 
 
 
 
3135
3136	/* no more funcs left */
3137	ftrace_shutdown(&trace_probe_ops, 0);
3138
3139	ftrace_probe_registered = 0;
3140}
3141
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3142
3143static void ftrace_free_entry(struct ftrace_func_probe *entry)
3144{
3145	if (entry->ops->free)
3146		entry->ops->free(entry->ops, entry->ip, &entry->data);
3147	kfree(entry);
 
 
3148}
3149
3150int
3151register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3152			      void *data)
 
3153{
3154	struct ftrace_func_probe *entry;
3155	struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
 
 
3156	struct ftrace_hash *hash;
3157	struct ftrace_page *pg;
3158	struct dyn_ftrace *rec;
3159	int type, len, not;
3160	unsigned long key;
3161	int count = 0;
3162	char *search;
3163	int ret;
 
3164
3165	type = filter_parse_regex(glob, strlen(glob), &search, &not);
3166	len = strlen(search);
3167
3168	/* we do not support '!' for function probes */
3169	if (WARN_ON(not))
3170		return -EINVAL;
3171
3172	mutex_lock(&trace_probe_ops.regex_lock);
3173
3174	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3175	if (!hash) {
3176		count = -ENOMEM;
3177		goto out;
 
 
 
3178	}
3179
3180	if (unlikely(ftrace_disabled)) {
3181		count = -ENODEV;
3182		goto out;
 
 
 
 
 
 
 
3183	}
3184
3185	mutex_lock(&ftrace_lock);
3186
3187	do_for_each_ftrace_rec(pg, rec) {
3188
3189		if (!ftrace_match_record(rec, NULL, search, len, type))
3190			continue;
 
 
 
3191
3192		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
3193		if (!entry) {
3194			/* If we did not process any, then return error */
3195			if (!count)
3196				count = -ENOMEM;
3197			goto out_unlock;
3198		}
3199
3200		count++;
 
 
 
3201
3202		entry->data = data;
3203
3204		/*
3205		 * The caller might want to do something special
3206		 * for each function we find. We call the callback
3207		 * to give the caller an opportunity to do so.
3208		 */
3209		if (ops->init) {
3210			if (ops->init(ops, rec->ip, &entry->data) < 0) {
3211				/* caller does not like this func */
3212				kfree(entry);
 
 
3213				continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3214			}
 
3215		}
 
3216
3217		ret = enter_record(hash, rec, 0);
3218		if (ret < 0) {
3219			kfree(entry);
3220			count = ret;
3221			goto out_unlock;
3222		}
3223
3224		entry->ops = ops;
3225		entry->ip = rec->ip;
3226
3227		key = hash_long(entry->ip, FTRACE_HASH_BITS);
3228		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
3229
3230	} while_for_each_ftrace_rec();
 
 
 
 
3231
3232	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
 
3233	if (ret < 0)
3234		count = ret;
 
 
 
3235
3236	__enable_ftrace_function_probe();
 
3237
3238 out_unlock:
3239	mutex_unlock(&ftrace_lock);
 
 
 
3240 out:
3241	mutex_unlock(&trace_probe_ops.regex_lock);
3242	free_ftrace_hash(hash);
3243
3244	return count;
3245}
3246
3247enum {
3248	PROBE_TEST_FUNC		= 1,
3249	PROBE_TEST_DATA		= 2
3250};
3251
3252static void
3253__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3254				  void *data, int flags)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3255{
3256	struct ftrace_func_entry *rec_entry;
3257	struct ftrace_func_probe *entry;
3258	struct ftrace_func_probe *p;
3259	struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
3260	struct list_head free_list;
3261	struct ftrace_hash *hash;
 
3262	struct hlist_node *tmp;
 
3263	char str[KSYM_SYMBOL_LEN];
3264	int type = MATCH_FULL;
3265	int i, len = 0;
3266	char *search;
3267
3268	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
3269		glob = NULL;
3270	else if (glob) {
3271		int not;
3272
3273		type = filter_parse_regex(glob, strlen(glob), &search, &not);
3274		len = strlen(search);
 
3275
3276		/* we do not support '!' for function probes */
3277		if (WARN_ON(not))
3278			return;
3279	}
3280
3281	mutex_lock(&trace_probe_ops.regex_lock);
 
 
 
 
 
 
 
 
 
3282
3283	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3284	if (!hash)
3285		/* Hmm, should report this somehow */
3286		goto out_unlock;
3287
3288	INIT_LIST_HEAD(&free_list);
3289
3290	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3291		struct hlist_head *hhd = &ftrace_func_hash[i];
3292
3293		hlist_for_each_entry_safe(entry, tmp, hhd, node) {
3294
3295			/* break up if statements for readability */
3296			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
3297				continue;
3298
3299			if ((flags & PROBE_TEST_DATA) && entry->data != data)
3300				continue;
3301
3302			/* do this last, since it is the most expensive */
3303			if (glob) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3304				kallsyms_lookup(entry->ip, NULL, NULL,
3305						NULL, str);
3306				if (!ftrace_match(str, glob, len, type))
3307					continue;
3308			}
3309
3310			rec_entry = ftrace_lookup_ip(hash, entry->ip);
3311			/* It is possible more than one entry had this ip */
3312			if (rec_entry)
3313				free_hash_entry(hash, rec_entry);
3314
3315			hlist_del_rcu(&entry->node);
3316			list_add(&entry->free_list, &free_list);
3317		}
3318	}
 
 
 
 
 
 
 
3319	mutex_lock(&ftrace_lock);
3320	__disable_ftrace_function_probe();
3321	/*
3322	 * Remove after the disable is called. Otherwise, if the last
3323	 * probe is removed, a null hash means *all enabled*.
3324	 */
3325	ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
3326	synchronize_sched();
3327	list_for_each_entry_safe(entry, p, &free_list, free_list) {
3328		list_del(&entry->free_list);
3329		ftrace_free_entry(entry);
 
 
 
 
 
 
 
 
 
 
 
 
3330	}
3331	mutex_unlock(&ftrace_lock);
3332		
3333 out_unlock:
3334	mutex_unlock(&trace_probe_ops.regex_lock);
3335	free_ftrace_hash(hash);
3336}
3337
3338void
3339unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3340				void *data)
3341{
3342	__unregister_ftrace_function_probe(glob, ops, data,
3343					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
3344}
3345
3346void
3347unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
3348{
3349	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
 
3350}
3351
3352void unregister_ftrace_function_probe_all(char *glob)
3353{
3354	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
 
 
 
3355}
3356
3357static LIST_HEAD(ftrace_commands);
3358static DEFINE_MUTEX(ftrace_cmd_mutex);
3359
3360/*
3361 * Currently we only register ftrace commands from __init, so mark this
3362 * __init too.
3363 */
3364__init int register_ftrace_command(struct ftrace_func_command *cmd)
3365{
3366	struct ftrace_func_command *p;
3367	int ret = 0;
3368
3369	mutex_lock(&ftrace_cmd_mutex);
3370	list_for_each_entry(p, &ftrace_commands, list) {
3371		if (strcmp(cmd->name, p->name) == 0) {
3372			ret = -EBUSY;
3373			goto out_unlock;
3374		}
3375	}
3376	list_add(&cmd->list, &ftrace_commands);
3377 out_unlock:
3378	mutex_unlock(&ftrace_cmd_mutex);
3379
3380	return ret;
3381}
3382
3383/*
3384 * Currently we only unregister ftrace commands from __init, so mark
3385 * this __init too.
3386 */
3387__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
3388{
3389	struct ftrace_func_command *p, *n;
3390	int ret = -ENODEV;
3391
3392	mutex_lock(&ftrace_cmd_mutex);
3393	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
3394		if (strcmp(cmd->name, p->name) == 0) {
3395			ret = 0;
3396			list_del_init(&p->list);
3397			goto out_unlock;
3398		}
3399	}
3400 out_unlock:
3401	mutex_unlock(&ftrace_cmd_mutex);
3402
3403	return ret;
3404}
3405
3406static int ftrace_process_regex(struct ftrace_hash *hash,
3407				char *buff, int len, int enable)
3408{
 
 
3409	char *func, *command, *next = buff;
3410	struct ftrace_func_command *p;
3411	int ret = -EINVAL;
3412
3413	func = strsep(&next, ":");
3414
3415	if (!next) {
3416		ret = ftrace_match_records(hash, func, len);
3417		if (!ret)
3418			ret = -EINVAL;
3419		if (ret < 0)
3420			return ret;
3421		return 0;
3422	}
3423
3424	/* command found */
3425
3426	command = strsep(&next, ":");
3427
3428	mutex_lock(&ftrace_cmd_mutex);
3429	list_for_each_entry(p, &ftrace_commands, list) {
3430		if (strcmp(p->name, command) == 0) {
3431			ret = p->func(hash, func, command, next, enable);
3432			goto out_unlock;
3433		}
3434	}
3435 out_unlock:
3436	mutex_unlock(&ftrace_cmd_mutex);
3437
3438	return ret;
3439}
3440
3441static ssize_t
3442ftrace_regex_write(struct file *file, const char __user *ubuf,
3443		   size_t cnt, loff_t *ppos, int enable)
3444{
3445	struct ftrace_iterator *iter;
3446	struct trace_parser *parser;
3447	ssize_t ret, read;
3448
3449	if (!cnt)
3450		return 0;
3451
3452	if (file->f_mode & FMODE_READ) {
3453		struct seq_file *m = file->private_data;
3454		iter = m->private;
3455	} else
3456		iter = file->private_data;
3457
3458	if (unlikely(ftrace_disabled))
3459		return -ENODEV;
3460
3461	/* iter->hash is a local copy, so we don't need regex_lock */
3462
3463	parser = &iter->parser;
3464	read = trace_get_user(parser, ubuf, cnt, ppos);
3465
3466	if (read >= 0 && trace_parser_loaded(parser) &&
3467	    !trace_parser_cont(parser)) {
3468		ret = ftrace_process_regex(iter->hash, parser->buffer,
3469					   parser->idx, enable);
3470		trace_parser_clear(parser);
3471		if (ret < 0)
3472			goto out;
3473	}
3474
3475	ret = read;
3476 out:
3477	return ret;
3478}
3479
3480ssize_t
3481ftrace_filter_write(struct file *file, const char __user *ubuf,
3482		    size_t cnt, loff_t *ppos)
3483{
3484	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
3485}
3486
3487ssize_t
3488ftrace_notrace_write(struct file *file, const char __user *ubuf,
3489		     size_t cnt, loff_t *ppos)
3490{
3491	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
3492}
3493
3494static int
3495ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
3496{
3497	struct ftrace_func_entry *entry;
3498
3499	if (!ftrace_location(ip))
 
3500		return -EINVAL;
3501
3502	if (remove) {
3503		entry = ftrace_lookup_ip(hash, ip);
3504		if (!entry)
3505			return -ENOENT;
3506		free_hash_entry(hash, entry);
3507		return 0;
3508	}
3509
3510	return add_hash_entry(hash, ip);
3511}
3512
3513static void ftrace_ops_update_code(struct ftrace_ops *ops)
 
 
3514{
3515	if (ops->flags & FTRACE_OPS_FL_ENABLED && ftrace_enabled)
3516		ftrace_run_update_code(FTRACE_UPDATE_CALLS);
 
 
 
 
 
 
 
 
 
 
 
 
3517}
3518
3519static int
3520ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
3521		unsigned long ip, int remove, int reset, int enable)
 
3522{
3523	struct ftrace_hash **orig_hash;
3524	struct ftrace_hash *hash;
3525	int ret;
3526
3527	/* All global ops uses the global ops filters */
3528	if (ops->flags & FTRACE_OPS_FL_GLOBAL)
3529		ops = &global_ops;
3530
3531	if (unlikely(ftrace_disabled))
3532		return -ENODEV;
3533
3534	mutex_lock(&ops->regex_lock);
3535
3536	if (enable)
3537		orig_hash = &ops->filter_hash;
 
 
 
 
 
3538	else
3539		orig_hash = &ops->notrace_hash;
3540
3541	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3542	if (!hash) {
3543		ret = -ENOMEM;
3544		goto out_regex_unlock;
3545	}
3546
3547	if (reset)
3548		ftrace_filter_reset(hash);
3549	if (buf && !ftrace_match_records(hash, buf, len)) {
3550		ret = -EINVAL;
3551		goto out_regex_unlock;
3552	}
3553	if (ip) {
3554		ret = ftrace_match_addr(hash, ip, remove);
3555		if (ret < 0)
3556			goto out_regex_unlock;
3557	}
3558
3559	mutex_lock(&ftrace_lock);
3560	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3561	if (!ret)
3562		ftrace_ops_update_code(ops);
3563
3564	mutex_unlock(&ftrace_lock);
3565
3566 out_regex_unlock:
3567	mutex_unlock(&ops->regex_lock);
3568
3569	free_ftrace_hash(hash);
3570	return ret;
3571}
3572
3573static int
3574ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
3575		int reset, int enable)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3576{
3577	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3578}
3579
3580/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3581 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
3582 * @ops - the ops to set the filter with
3583 * @ip - the address to add to or remove from the filter.
3584 * @remove - non zero to remove the ip from the filter
3585 * @reset - non zero to reset all filters before applying this filter.
3586 *
3587 * Filters denote which functions should be enabled when tracing is enabled
3588 * If @ip is NULL, it failes to update filter.
 
 
 
 
3589 */
3590int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
3591			 int remove, int reset)
3592{
3593	ftrace_ops_init(ops);
3594	return ftrace_set_addr(ops, ip, remove, reset, 1);
3595}
3596EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
3597
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3598static int
3599ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
3600		 int reset, int enable)
3601{
3602	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
3603}
3604
3605/**
3606 * ftrace_set_filter - set a function to filter on in ftrace
3607 * @ops - the ops to set the filter with
3608 * @buf - the string that holds the function filter text.
3609 * @len - the length of the string.
3610 * @reset - non zero to reset all filters before applying this filter.
3611 *
3612 * Filters denote which functions should be enabled when tracing is enabled.
3613 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 
 
 
 
3614 */
3615int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
3616		       int len, int reset)
3617{
3618	ftrace_ops_init(ops);
3619	return ftrace_set_regex(ops, buf, len, reset, 1);
3620}
3621EXPORT_SYMBOL_GPL(ftrace_set_filter);
3622
3623/**
3624 * ftrace_set_notrace - set a function to not trace in ftrace
3625 * @ops - the ops to set the notrace filter with
3626 * @buf - the string that holds the function notrace text.
3627 * @len - the length of the string.
3628 * @reset - non zero to reset all filters before applying this filter.
3629 *
3630 * Notrace Filters denote which functions should not be enabled when tracing
3631 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3632 * for tracing.
 
 
 
 
3633 */
3634int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
3635			int len, int reset)
3636{
3637	ftrace_ops_init(ops);
3638	return ftrace_set_regex(ops, buf, len, reset, 0);
3639}
3640EXPORT_SYMBOL_GPL(ftrace_set_notrace);
3641/**
3642 * ftrace_set_filter - set a function to filter on in ftrace
3643 * @ops - the ops to set the filter with
3644 * @buf - the string that holds the function filter text.
3645 * @len - the length of the string.
3646 * @reset - non zero to reset all filters before applying this filter.
3647 *
3648 * Filters denote which functions should be enabled when tracing is enabled.
3649 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
3650 */
3651void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
3652{
3653	ftrace_set_regex(&global_ops, buf, len, reset, 1);
3654}
3655EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
3656
3657/**
3658 * ftrace_set_notrace - set a function to not trace in ftrace
3659 * @ops - the ops to set the notrace filter with
3660 * @buf - the string that holds the function notrace text.
3661 * @len - the length of the string.
3662 * @reset - non zero to reset all filters before applying this filter.
3663 *
3664 * Notrace Filters denote which functions should not be enabled when tracing
3665 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3666 * for tracing.
3667 */
3668void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
3669{
3670	ftrace_set_regex(&global_ops, buf, len, reset, 0);
3671}
3672EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
3673
3674/*
3675 * command line interface to allow users to set filters on boot up.
3676 */
3677#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
3678static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
3679static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
3680
3681/* Used by function selftest to not test if filter is set */
3682bool ftrace_filter_param __initdata;
3683
3684static int __init set_ftrace_notrace(char *str)
3685{
3686	ftrace_filter_param = true;
3687	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
3688	return 1;
3689}
3690__setup("ftrace_notrace=", set_ftrace_notrace);
3691
3692static int __init set_ftrace_filter(char *str)
3693{
3694	ftrace_filter_param = true;
3695	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
3696	return 1;
3697}
3698__setup("ftrace_filter=", set_ftrace_filter);
3699
3700#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3701static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
3702static int ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer);
 
3703
3704static int __init set_graph_function(char *str)
3705{
3706	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
3707	return 1;
3708}
3709__setup("ftrace_graph_filter=", set_graph_function);
3710
3711static void __init set_ftrace_early_graph(char *buf)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3712{
3713	int ret;
3714	char *func;
 
 
 
 
 
3715
3716	while (buf) {
3717		func = strsep(&buf, ",");
3718		/* we allow only one expression at a time */
3719		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3720				      FTRACE_GRAPH_MAX_FUNCS, func);
3721		if (ret)
3722			printk(KERN_DEBUG "ftrace: function %s not "
3723					  "traceable\n", func);
3724	}
 
 
 
 
 
3725}
3726#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3727
3728void __init
3729ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
3730{
3731	char *func;
3732
3733	ftrace_ops_init(ops);
3734
3735	while (buf) {
3736		func = strsep(&buf, ",");
3737		ftrace_set_regex(ops, func, strlen(func), 0, enable);
3738	}
3739}
3740
3741static void __init set_ftrace_early_filters(void)
3742{
3743	if (ftrace_filter_buf[0])
3744		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
3745	if (ftrace_notrace_buf[0])
3746		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
3747#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3748	if (ftrace_graph_buf[0])
3749		set_ftrace_early_graph(ftrace_graph_buf);
 
 
3750#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3751}
3752
3753int ftrace_regex_release(struct inode *inode, struct file *file)
3754{
3755	struct seq_file *m = (struct seq_file *)file->private_data;
3756	struct ftrace_iterator *iter;
3757	struct ftrace_hash **orig_hash;
3758	struct trace_parser *parser;
3759	int filter_hash;
3760	int ret;
3761
3762	if (file->f_mode & FMODE_READ) {
3763		iter = m->private;
3764		seq_release(inode, file);
3765	} else
3766		iter = file->private_data;
3767
3768	parser = &iter->parser;
3769	if (trace_parser_loaded(parser)) {
3770		parser->buffer[parser->idx] = 0;
3771		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
 
 
3772	}
3773
3774	trace_parser_put(parser);
3775
3776	mutex_lock(&iter->ops->regex_lock);
3777
3778	if (file->f_mode & FMODE_WRITE) {
3779		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3780
3781		if (filter_hash)
3782			orig_hash = &iter->ops->filter_hash;
3783		else
3784			orig_hash = &iter->ops->notrace_hash;
 
 
 
 
 
 
3785
3786		mutex_lock(&ftrace_lock);
3787		ret = ftrace_hash_move(iter->ops, filter_hash,
3788				       orig_hash, iter->hash);
3789		if (!ret)
3790			ftrace_ops_update_code(iter->ops);
3791
3792		mutex_unlock(&ftrace_lock);
 
 
 
3793	}
3794
3795	mutex_unlock(&iter->ops->regex_lock);
3796	free_ftrace_hash(iter->hash);
 
 
3797	kfree(iter);
3798
3799	return 0;
3800}
3801
3802static const struct file_operations ftrace_avail_fops = {
3803	.open = ftrace_avail_open,
3804	.read = seq_read,
3805	.llseek = seq_lseek,
3806	.release = seq_release_private,
3807};
3808
3809static const struct file_operations ftrace_enabled_fops = {
3810	.open = ftrace_enabled_open,
3811	.read = seq_read,
3812	.llseek = seq_lseek,
3813	.release = seq_release_private,
3814};
3815
3816static const struct file_operations ftrace_filter_fops = {
3817	.open = ftrace_filter_open,
3818	.read = seq_read,
3819	.write = ftrace_filter_write,
3820	.llseek = tracing_lseek,
3821	.release = ftrace_regex_release,
3822};
3823
3824static const struct file_operations ftrace_notrace_fops = {
3825	.open = ftrace_notrace_open,
3826	.read = seq_read,
3827	.write = ftrace_notrace_write,
3828	.llseek = tracing_lseek,
3829	.release = ftrace_regex_release,
3830};
3831
3832#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3833
3834static DEFINE_MUTEX(graph_lock);
3835
3836int ftrace_graph_count;
3837int ftrace_graph_notrace_count;
3838unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3839unsigned long ftrace_graph_notrace_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
 
 
 
 
 
3840
3841struct ftrace_graph_data {
3842	unsigned long *table;
3843	size_t size;
3844	int *count;
3845	const struct seq_operations *seq_ops;
 
 
 
3846};
3847
3848static void *
3849__g_next(struct seq_file *m, loff_t *pos)
3850{
3851	struct ftrace_graph_data *fgd = m->private;
 
 
 
3852
3853	if (*pos >= *fgd->count)
3854		return NULL;
3855	return &fgd->table[*pos];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3856}
3857
3858static void *
3859g_next(struct seq_file *m, void *v, loff_t *pos)
3860{
3861	(*pos)++;
3862	return __g_next(m, pos);
3863}
3864
3865static void *g_start(struct seq_file *m, loff_t *pos)
3866{
3867	struct ftrace_graph_data *fgd = m->private;
3868
3869	mutex_lock(&graph_lock);
3870
 
 
 
 
 
 
 
3871	/* Nothing, tell g_show to print all functions are enabled */
3872	if (!*fgd->count && !*pos)
3873		return (void *)1;
3874
 
 
3875	return __g_next(m, pos);
3876}
3877
3878static void g_stop(struct seq_file *m, void *p)
3879{
3880	mutex_unlock(&graph_lock);
3881}
3882
3883static int g_show(struct seq_file *m, void *v)
3884{
3885	unsigned long *ptr = v;
3886
3887	if (!ptr)
3888		return 0;
3889
3890	if (ptr == (unsigned long *)1) {
3891		seq_printf(m, "#### all functions enabled ####\n");
 
 
 
 
 
3892		return 0;
3893	}
3894
3895	seq_printf(m, "%ps\n", (void *)*ptr);
3896
3897	return 0;
3898}
3899
3900static const struct seq_operations ftrace_graph_seq_ops = {
3901	.start = g_start,
3902	.next = g_next,
3903	.stop = g_stop,
3904	.show = g_show,
3905};
3906
3907static int
3908__ftrace_graph_open(struct inode *inode, struct file *file,
3909		    struct ftrace_graph_data *fgd)
3910{
3911	int ret = 0;
 
3912
3913	mutex_lock(&graph_lock);
3914	if ((file->f_mode & FMODE_WRITE) &&
3915	    (file->f_flags & O_TRUNC)) {
3916		*fgd->count = 0;
3917		memset(fgd->table, 0, fgd->size * sizeof(*fgd->table));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3918	}
3919	mutex_unlock(&graph_lock);
3920
3921	if (file->f_mode & FMODE_READ) {
3922		ret = seq_open(file, fgd->seq_ops);
3923		if (!ret) {
3924			struct seq_file *m = file->private_data;
3925			m->private = fgd;
 
 
 
 
3926		}
3927	} else
3928		file->private_data = fgd;
3929
 
 
 
 
 
 
 
 
 
 
 
 
 
3930	return ret;
3931}
3932
3933static int
3934ftrace_graph_open(struct inode *inode, struct file *file)
3935{
3936	struct ftrace_graph_data *fgd;
 
3937
3938	if (unlikely(ftrace_disabled))
3939		return -ENODEV;
3940
3941	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
3942	if (fgd == NULL)
3943		return -ENOMEM;
3944
3945	fgd->table = ftrace_graph_funcs;
3946	fgd->size = FTRACE_GRAPH_MAX_FUNCS;
3947	fgd->count = &ftrace_graph_count;
 
 
3948	fgd->seq_ops = &ftrace_graph_seq_ops;
3949
3950	return __ftrace_graph_open(inode, file, fgd);
 
 
 
 
 
3951}
3952
3953static int
3954ftrace_graph_notrace_open(struct inode *inode, struct file *file)
3955{
3956	struct ftrace_graph_data *fgd;
 
3957
3958	if (unlikely(ftrace_disabled))
3959		return -ENODEV;
3960
3961	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
3962	if (fgd == NULL)
3963		return -ENOMEM;
3964
3965	fgd->table = ftrace_graph_notrace_funcs;
3966	fgd->size = FTRACE_GRAPH_MAX_FUNCS;
3967	fgd->count = &ftrace_graph_notrace_count;
 
 
3968	fgd->seq_ops = &ftrace_graph_seq_ops;
3969
3970	return __ftrace_graph_open(inode, file, fgd);
 
 
 
 
 
3971}
3972
3973static int
3974ftrace_graph_release(struct inode *inode, struct file *file)
3975{
 
 
 
 
 
3976	if (file->f_mode & FMODE_READ) {
3977		struct seq_file *m = file->private_data;
3978
3979		kfree(m->private);
3980		seq_release(inode, file);
3981	} else {
3982		kfree(file->private_data);
3983	}
3984
3985	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3986}
3987
3988static int
3989ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer)
3990{
 
3991	struct dyn_ftrace *rec;
3992	struct ftrace_page *pg;
3993	int search_len;
3994	int fail = 1;
3995	int type, not;
3996	char *search;
3997	bool exists;
3998	int i;
3999
4000	/* decode regex */
4001	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
4002	if (!not && *idx >= size)
4003		return -EBUSY;
4004
4005	search_len = strlen(search);
4006
4007	mutex_lock(&ftrace_lock);
4008
4009	if (unlikely(ftrace_disabled)) {
4010		mutex_unlock(&ftrace_lock);
4011		return -ENODEV;
4012	}
4013
4014	do_for_each_ftrace_rec(pg, rec) {
4015
4016		if (ftrace_match_record(rec, NULL, search, search_len, type)) {
4017			/* if it is in the array */
4018			exists = false;
4019			for (i = 0; i < *idx; i++) {
4020				if (array[i] == rec->ip) {
4021					exists = true;
4022					break;
4023				}
4024			}
4025
4026			if (!not) {
4027				fail = 0;
4028				if (!exists) {
4029					array[(*idx)++] = rec->ip;
4030					if (*idx >= size)
4031						goto out;
4032				}
4033			} else {
4034				if (exists) {
4035					array[i] = array[--(*idx)];
4036					array[*idx] = 0;
4037					fail = 0;
4038				}
4039			}
4040		}
4041	} while_for_each_ftrace_rec();
4042out:
4043	mutex_unlock(&ftrace_lock);
4044
4045	if (fail)
4046		return -EINVAL;
4047
4048	return 0;
4049}
4050
4051static ssize_t
4052ftrace_graph_write(struct file *file, const char __user *ubuf,
4053		   size_t cnt, loff_t *ppos)
4054{
4055	struct trace_parser parser;
4056	ssize_t read, ret = 0;
4057	struct ftrace_graph_data *fgd = file->private_data;
 
4058
4059	if (!cnt)
4060		return 0;
4061
4062	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX))
4063		return -ENOMEM;
4064
4065	read = trace_get_user(&parser, ubuf, cnt, ppos);
 
4066
4067	if (read >= 0 && trace_parser_loaded((&parser))) {
4068		parser.buffer[parser.idx] = 0;
4069
4070		mutex_lock(&graph_lock);
4071
4072		/* we allow only one expression at a time */
4073		ret = ftrace_set_func(fgd->table, fgd->count, fgd->size,
4074				      parser.buffer);
4075
4076		mutex_unlock(&graph_lock);
 
 
4077	}
4078
4079	if (!ret)
4080		ret = read;
4081
4082	trace_parser_put(&parser);
4083
4084	return ret;
4085}
4086
4087static const struct file_operations ftrace_graph_fops = {
4088	.open		= ftrace_graph_open,
4089	.read		= seq_read,
4090	.write		= ftrace_graph_write,
4091	.llseek		= tracing_lseek,
4092	.release	= ftrace_graph_release,
4093};
4094
4095static const struct file_operations ftrace_graph_notrace_fops = {
4096	.open		= ftrace_graph_notrace_open,
4097	.read		= seq_read,
4098	.write		= ftrace_graph_write,
4099	.llseek		= tracing_lseek,
4100	.release	= ftrace_graph_release,
4101};
4102#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4103
4104void ftrace_create_filter_files(struct ftrace_ops *ops,
4105				struct dentry *parent)
4106{
4107
4108	trace_create_file("set_ftrace_filter", 0644, parent,
4109			  ops, &ftrace_filter_fops);
4110
4111	trace_create_file("set_ftrace_notrace", 0644, parent,
4112			  ops, &ftrace_notrace_fops);
4113}
4114
4115/*
4116 * The name "destroy_filter_files" is really a misnomer. Although
4117 * in the future, it may actualy delete the files, but this is
4118 * really intended to make sure the ops passed in are disabled
4119 * and that when this function returns, the caller is free to
4120 * free the ops.
4121 *
4122 * The "destroy" name is only to match the "create" name that this
4123 * should be paired with.
4124 */
4125void ftrace_destroy_filter_files(struct ftrace_ops *ops)
4126{
4127	mutex_lock(&ftrace_lock);
4128	if (ops->flags & FTRACE_OPS_FL_ENABLED)
4129		ftrace_shutdown(ops, 0);
4130	ops->flags |= FTRACE_OPS_FL_DELETED;
 
4131	mutex_unlock(&ftrace_lock);
4132}
4133
4134static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
4135{
4136
4137	trace_create_file("available_filter_functions", 0444,
4138			d_tracer, NULL, &ftrace_avail_fops);
4139
4140	trace_create_file("enabled_functions", 0444,
4141			d_tracer, NULL, &ftrace_enabled_fops);
4142
4143	ftrace_create_filter_files(&global_ops, d_tracer);
4144
4145#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4146	trace_create_file("set_graph_function", 0444, d_tracer,
4147				    NULL,
4148				    &ftrace_graph_fops);
4149	trace_create_file("set_graph_notrace", 0444, d_tracer,
4150				    NULL,
4151				    &ftrace_graph_notrace_fops);
4152#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4153
4154	return 0;
4155}
4156
4157static int ftrace_cmp_ips(const void *a, const void *b)
4158{
4159	const unsigned long *ipa = a;
4160	const unsigned long *ipb = b;
4161
4162	if (*ipa > *ipb)
4163		return 1;
4164	if (*ipa < *ipb)
4165		return -1;
4166	return 0;
4167}
4168
4169static void ftrace_swap_ips(void *a, void *b, int size)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4170{
4171	unsigned long *ipa = a;
4172	unsigned long *ipb = b;
4173	unsigned long t;
4174
4175	t = *ipa;
4176	*ipa = *ipb;
4177	*ipb = t;
4178}
 
4179
4180static int ftrace_process_locs(struct module *mod,
4181			       unsigned long *start,
4182			       unsigned long *end)
4183{
4184	struct ftrace_page *start_pg;
4185	struct ftrace_page *pg;
4186	struct dyn_ftrace *rec;
4187	unsigned long count;
4188	unsigned long *p;
4189	unsigned long addr;
4190	unsigned long flags = 0; /* Shut up gcc */
4191	int ret = -ENOMEM;
4192
4193	count = end - start;
4194
4195	if (!count)
4196		return 0;
4197
4198	sort(start, count, sizeof(*start),
4199	     ftrace_cmp_ips, ftrace_swap_ips);
 
 
 
 
 
 
 
 
 
4200
4201	start_pg = ftrace_allocate_pages(count);
4202	if (!start_pg)
4203		return -ENOMEM;
4204
4205	mutex_lock(&ftrace_lock);
4206
4207	/*
4208	 * Core and each module needs their own pages, as
4209	 * modules will free them when they are removed.
4210	 * Force a new page to be allocated for modules.
4211	 */
4212	if (!mod) {
4213		WARN_ON(ftrace_pages || ftrace_pages_start);
4214		/* First initialization */
4215		ftrace_pages = ftrace_pages_start = start_pg;
4216	} else {
4217		if (!ftrace_pages)
4218			goto out;
4219
4220		if (WARN_ON(ftrace_pages->next)) {
4221			/* Hmm, we have free pages? */
4222			while (ftrace_pages->next)
4223				ftrace_pages = ftrace_pages->next;
4224		}
4225
4226		ftrace_pages->next = start_pg;
4227	}
4228
4229	p = start;
4230	pg = start_pg;
4231	while (p < end) {
 
4232		addr = ftrace_call_adjust(*p++);
4233		/*
4234		 * Some architecture linkers will pad between
4235		 * the different mcount_loc sections of different
4236		 * object files to satisfy alignments.
4237		 * Skip any NULL pointers.
4238		 */
4239		if (!addr)
4240			continue;
4241
4242		if (pg->index == pg->size) {
 
4243			/* We should have allocated enough */
4244			if (WARN_ON(!pg->next))
4245				break;
4246			pg = pg->next;
4247		}
4248
4249		rec = &pg->records[pg->index++];
4250		rec->ip = addr;
4251	}
4252
4253	/* We should have used all pages */
4254	WARN_ON(pg->next);
4255
4256	/* Assign the last page to ftrace_pages */
4257	ftrace_pages = pg;
4258
4259	/*
4260	 * We only need to disable interrupts on start up
4261	 * because we are modifying code that an interrupt
4262	 * may execute, and the modification is not atomic.
4263	 * But for modules, nothing runs the code we modify
4264	 * until we are finished with it, and there's no
4265	 * reason to cause large interrupt latencies while we do it.
4266	 */
4267	if (!mod)
4268		local_irq_save(flags);
4269	ftrace_update_code(mod, start_pg);
4270	if (!mod)
4271		local_irq_restore(flags);
4272	ret = 0;
4273 out:
4274	mutex_unlock(&ftrace_lock);
4275
4276	return ret;
4277}
4278
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4279#ifdef CONFIG_MODULES
4280
4281#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
4282
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4283void ftrace_release_mod(struct module *mod)
4284{
 
 
4285	struct dyn_ftrace *rec;
4286	struct ftrace_page **last_pg;
 
4287	struct ftrace_page *pg;
4288	int order;
4289
4290	mutex_lock(&ftrace_lock);
4291
4292	if (ftrace_disabled)
4293		goto out_unlock;
4294
 
 
 
 
 
 
 
 
4295	/*
4296	 * Each module has its own ftrace_pages, remove
4297	 * them from the list.
4298	 */
4299	last_pg = &ftrace_pages_start;
4300	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
4301		rec = &pg->records[0];
4302		if (within_module_core(rec->ip, mod)) {
 
4303			/*
4304			 * As core pages are first, the first
4305			 * page should never be a module page.
4306			 */
4307			if (WARN_ON(pg == ftrace_pages_start))
4308				goto out_unlock;
4309
4310			/* Check if we are deleting the last page */
4311			if (pg == ftrace_pages)
4312				ftrace_pages = next_to_ftrace_page(last_pg);
4313
 
4314			*last_pg = pg->next;
4315			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
4316			free_pages((unsigned long)pg->records, order);
4317			kfree(pg);
4318		} else
4319			last_pg = &pg->next;
4320	}
4321 out_unlock:
4322	mutex_unlock(&ftrace_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4323}
4324
4325static void ftrace_init_module(struct module *mod,
4326			       unsigned long *start, unsigned long *end)
4327{
4328	if (ftrace_disabled || start == end)
4329		return;
4330	ftrace_process_locs(mod, start, end);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4331}
4332
4333void ftrace_module_init(struct module *mod)
4334{
4335	ftrace_init_module(mod, mod->ftrace_callsites,
4336			   mod->ftrace_callsites +
4337			   mod->num_ftrace_callsites);
 
 
 
 
 
 
 
4338}
4339
4340static int ftrace_module_notify_exit(struct notifier_block *self,
4341				     unsigned long val, void *data)
4342{
4343	struct module *mod = data;
 
 
 
 
 
4344
4345	if (val == MODULE_STATE_GOING)
4346		ftrace_release_mod(mod);
 
4347
4348	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4349}
 
4350#else
4351static int ftrace_module_notify_exit(struct notifier_block *self,
4352				     unsigned long val, void *data)
 
 
 
4353{
4354	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
4355}
4356#endif /* CONFIG_MODULES */
4357
4358struct notifier_block ftrace_module_exit_nb = {
4359	.notifier_call = ftrace_module_notify_exit,
4360	.priority = INT_MIN,	/* Run after anything that can remove kprobes */
4361};
4362
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4363void __init ftrace_init(void)
4364{
4365	extern unsigned long __start_mcount_loc[];
4366	extern unsigned long __stop_mcount_loc[];
4367	unsigned long count, flags;
4368	int ret;
4369
4370	local_irq_save(flags);
4371	ret = ftrace_dyn_arch_init();
4372	local_irq_restore(flags);
4373	if (ret)
4374		goto failed;
4375
4376	count = __stop_mcount_loc - __start_mcount_loc;
4377	if (!count) {
4378		pr_info("ftrace: No functions to be traced?\n");
4379		goto failed;
4380	}
4381
4382	pr_info("ftrace: allocating %ld entries in %ld pages\n",
4383		count, count / ENTRIES_PER_PAGE + 1);
4384
4385	last_ftrace_enabled = ftrace_enabled = 1;
4386
4387	ret = ftrace_process_locs(NULL,
4388				  __start_mcount_loc,
4389				  __stop_mcount_loc);
 
 
 
 
4390
4391	ret = register_module_notifier(&ftrace_module_exit_nb);
4392	if (ret)
4393		pr_warning("Failed to register trace ftrace module exit notifier\n");
 
4394
4395	set_ftrace_early_filters();
4396
4397	return;
4398 failed:
4399	ftrace_disabled = 1;
4400}
4401
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4402#else
4403
4404static struct ftrace_ops global_ops = {
4405	.func			= ftrace_stub,
4406	.flags			= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
4407	INIT_REGEX_LOCK(global_ops)
4408};
4409
4410static int __init ftrace_nodyn_init(void)
4411{
4412	ftrace_enabled = 1;
4413	return 0;
4414}
4415core_initcall(ftrace_nodyn_init);
4416
4417static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
4418static inline void ftrace_startup_enable(int command) { }
4419/* Keep as macros so we do not need to define the commands */
4420# define ftrace_startup(ops, command)					\
4421	({								\
4422		int ___ret = __register_ftrace_function(ops);		\
4423		if (!___ret)						\
4424			(ops)->flags |= FTRACE_OPS_FL_ENABLED;		\
4425		___ret;							\
4426	})
4427# define ftrace_shutdown(ops, command)					\
4428	({								\
4429		int ___ret = __unregister_ftrace_function(ops);		\
4430		if (!___ret)						\
4431			(ops)->flags &= ~FTRACE_OPS_FL_ENABLED;		\
4432		___ret;							\
4433	})
4434
4435# define ftrace_startup_sysctl()	do { } while (0)
4436# define ftrace_shutdown_sysctl()	do { } while (0)
4437
4438static inline int
4439ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
4440{
4441	return 1;
4442}
4443
4444#endif /* CONFIG_DYNAMIC_FTRACE */
4445
4446static void
4447ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
4448			struct ftrace_ops *op, struct pt_regs *regs)
4449{
4450	if (unlikely(trace_recursion_test(TRACE_CONTROL_BIT)))
4451		return;
4452
4453	/*
4454	 * Some of the ops may be dynamically allocated,
4455	 * they must be freed after a synchronize_sched().
4456	 */
4457	preempt_disable_notrace();
4458	trace_recursion_set(TRACE_CONTROL_BIT);
4459
4460	/*
4461	 * Control funcs (perf) uses RCU. Only trace if
4462	 * RCU is currently active.
4463	 */
4464	if (!rcu_is_watching())
4465		goto out;
4466
4467	do_for_each_ftrace_op(op, ftrace_control_list) {
4468		if (!(op->flags & FTRACE_OPS_FL_STUB) &&
4469		    !ftrace_function_local_disabled(op) &&
4470		    ftrace_ops_test(op, ip, regs))
4471			op->func(ip, parent_ip, op, regs);
4472	} while_for_each_ftrace_op(op);
4473 out:
4474	trace_recursion_clear(TRACE_CONTROL_BIT);
4475	preempt_enable_notrace();
 
4476}
4477
4478static struct ftrace_ops control_ops = {
4479	.func	= ftrace_ops_control_func,
4480	.flags	= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
4481	INIT_REGEX_LOCK(control_ops)
4482};
4483
4484static inline void
4485__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
4486		       struct ftrace_ops *ignored, struct pt_regs *regs)
4487{
 
4488	struct ftrace_ops *op;
4489	int bit;
4490
4491	if (function_trace_stop)
4492		return;
4493
4494	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
 
 
4495	if (bit < 0)
4496		return;
4497
4498	/*
4499	 * Some of the ops may be dynamically allocated,
4500	 * they must be freed after a synchronize_sched().
4501	 */
4502	preempt_disable_notrace();
4503	do_for_each_ftrace_op(op, ftrace_ops_list) {
4504		if (ftrace_ops_test(op, ip, regs))
4505			op->func(ip, parent_ip, op, regs);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4506	} while_for_each_ftrace_op(op);
4507	preempt_enable_notrace();
4508	trace_clear_recursion(bit);
4509}
4510
4511/*
4512 * Some archs only support passing ip and parent_ip. Even though
4513 * the list function ignores the op parameter, we do not want any
4514 * C side effects, where a function is called without the caller
4515 * sending a third parameter.
4516 * Archs are to support both the regs and ftrace_ops at the same time.
4517 * If they support ftrace_ops, it is assumed they support regs.
4518 * If call backs want to use regs, they must either check for regs
4519 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
4520 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
4521 * An architecture can pass partial regs with ftrace_ops and still
4522 * set the ARCH_SUPPORT_FTARCE_OPS.
 
 
 
4523 */
4524#if ARCH_SUPPORTS_FTRACE_OPS
4525static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
4526				 struct ftrace_ops *op, struct pt_regs *regs)
4527{
4528	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
4529}
4530#else
4531static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
4532{
4533	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
4534}
4535#endif
 
4536
4537static void clear_ftrace_swapper(void)
 
 
 
 
 
 
4538{
4539	struct task_struct *p;
4540	int cpu;
4541
4542	get_online_cpus();
4543	for_each_online_cpu(cpu) {
4544		p = idle_task(cpu);
4545		clear_tsk_trace_trace(p);
4546	}
4547	put_online_cpus();
4548}
4549
4550static void set_ftrace_swapper(void)
4551{
4552	struct task_struct *p;
4553	int cpu;
4554
4555	get_online_cpus();
4556	for_each_online_cpu(cpu) {
4557		p = idle_task(cpu);
4558		set_tsk_trace_trace(p);
4559	}
4560	put_online_cpus();
4561}
 
4562
4563static void clear_ftrace_pid(struct pid *pid)
 
 
 
 
 
 
 
 
 
 
 
4564{
4565	struct task_struct *p;
 
 
 
 
 
 
4566
4567	rcu_read_lock();
4568	do_each_pid_task(pid, PIDTYPE_PID, p) {
4569		clear_tsk_trace_trace(p);
4570	} while_each_pid_task(pid, PIDTYPE_PID, p);
4571	rcu_read_unlock();
4572
4573	put_pid(pid);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4574}
4575
4576static void set_ftrace_pid(struct pid *pid)
 
 
 
4577{
4578	struct task_struct *p;
 
4579
4580	rcu_read_lock();
4581	do_each_pid_task(pid, PIDTYPE_PID, p) {
4582		set_tsk_trace_trace(p);
4583	} while_each_pid_task(pid, PIDTYPE_PID, p);
4584	rcu_read_unlock();
4585}
4586
4587static void clear_ftrace_pid_task(struct pid *pid)
 
4588{
4589	if (pid == ftrace_swapper_pid)
4590		clear_ftrace_swapper();
4591	else
4592		clear_ftrace_pid(pid);
 
 
 
 
4593}
4594
4595static void set_ftrace_pid_task(struct pid *pid)
4596{
4597	if (pid == ftrace_swapper_pid)
4598		set_ftrace_swapper();
4599	else
4600		set_ftrace_pid(pid);
 
 
 
 
 
 
 
4601}
4602
4603static int ftrace_pid_add(int p)
4604{
4605	struct pid *pid;
4606	struct ftrace_pid *fpid;
4607	int ret = -EINVAL;
4608
4609	mutex_lock(&ftrace_lock);
4610
4611	if (!p)
4612		pid = ftrace_swapper_pid;
4613	else
4614		pid = find_get_pid(p);
4615
4616	if (!pid)
4617		goto out;
 
 
4618
4619	ret = 0;
 
 
4620
4621	list_for_each_entry(fpid, &ftrace_pids, list)
4622		if (fpid->pid == pid)
4623			goto out_put;
 
 
 
4624
4625	ret = -ENOMEM;
 
4626
4627	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
4628	if (!fpid)
4629		goto out_put;
4630
4631	list_add(&fpid->list, &ftrace_pids);
4632	fpid->pid = pid;
4633
4634	set_ftrace_pid_task(pid);
 
4635
4636	ftrace_update_pid_func();
4637	ftrace_startup_enable(0);
 
4638
4639	mutex_unlock(&ftrace_lock);
4640	return 0;
 
4641
4642out_put:
4643	if (pid != ftrace_swapper_pid)
4644		put_pid(pid);
4645
4646out:
4647	mutex_unlock(&ftrace_lock);
4648	return ret;
4649}
4650
4651static void ftrace_pid_reset(void)
4652{
4653	struct ftrace_pid *fpid, *safe;
4654
4655	mutex_lock(&ftrace_lock);
4656	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
4657		struct pid *pid = fpid->pid;
4658
4659		clear_ftrace_pid_task(pid);
4660
4661		list_del(&fpid->list);
4662		kfree(fpid);
4663	}
4664
4665	ftrace_update_pid_func();
4666	ftrace_startup_enable(0);
4667
4668	mutex_unlock(&ftrace_lock);
4669}
4670
 
 
 
4671static void *fpid_start(struct seq_file *m, loff_t *pos)
 
4672{
 
 
 
4673	mutex_lock(&ftrace_lock);
 
 
 
4674
4675	if (list_empty(&ftrace_pids) && (!*pos))
4676		return (void *) 1;
4677
4678	return seq_list_start(&ftrace_pids, *pos);
4679}
4680
4681static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
4682{
4683	if (v == (void *)1)
4684		return NULL;
4685
4686	return seq_list_next(v, &ftrace_pids, pos);
 
 
 
 
4687}
4688
4689static void fpid_stop(struct seq_file *m, void *p)
 
4690{
 
4691	mutex_unlock(&ftrace_lock);
4692}
4693
4694static int fpid_show(struct seq_file *m, void *v)
4695{
4696	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
4697
4698	if (v == (void *)1) {
4699		seq_printf(m, "no pid\n");
4700		return 0;
4701	}
4702
4703	if (fpid->pid == ftrace_swapper_pid)
4704		seq_printf(m, "swapper tasks\n");
4705	else
4706		seq_printf(m, "%u\n", pid_vnr(fpid->pid));
4707
4708	return 0;
4709}
4710
4711static const struct seq_operations ftrace_pid_sops = {
4712	.start = fpid_start,
4713	.next = fpid_next,
4714	.stop = fpid_stop,
4715	.show = fpid_show,
4716};
4717
4718static int
4719ftrace_pid_open(struct inode *inode, struct file *file)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4720{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4721	int ret = 0;
4722
 
 
 
 
4723	if ((file->f_mode & FMODE_WRITE) &&
4724	    (file->f_flags & O_TRUNC))
4725		ftrace_pid_reset();
 
 
 
 
 
 
 
 
 
 
 
 
 
4726
4727	if (file->f_mode & FMODE_READ)
4728		ret = seq_open(file, &ftrace_pid_sops);
 
 
 
 
 
 
4729
4730	return ret;
4731}
4732
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4733static ssize_t
4734ftrace_pid_write(struct file *filp, const char __user *ubuf,
4735		   size_t cnt, loff_t *ppos)
4736{
4737	char buf[64], *tmp;
4738	long val;
4739	int ret;
 
 
 
4740
4741	if (cnt >= sizeof(buf))
4742		return -EINVAL;
 
 
4743
4744	if (copy_from_user(&buf, ubuf, cnt))
4745		return -EFAULT;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4746
4747	buf[cnt] = 0;
 
 
 
 
 
 
 
4748
4749	/*
4750	 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
4751	 * to clean the filter quietly.
 
4752	 */
4753	tmp = strstrip(buf);
4754	if (strlen(tmp) == 0)
4755		return 1;
4756
4757	ret = kstrtol(tmp, 10, &val);
4758	if (ret < 0)
4759		return ret;
 
4760
4761	ret = ftrace_pid_add(val);
 
4762
4763	return ret ? ret : cnt;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4764}
4765
4766static int
4767ftrace_pid_release(struct inode *inode, struct file *file)
4768{
4769	if (file->f_mode & FMODE_READ)
4770		seq_release(inode, file);
4771
4772	return 0;
 
 
4773}
4774
4775static const struct file_operations ftrace_pid_fops = {
4776	.open		= ftrace_pid_open,
4777	.write		= ftrace_pid_write,
4778	.read		= seq_read,
4779	.llseek		= tracing_lseek,
4780	.release	= ftrace_pid_release,
4781};
4782
4783static __init int ftrace_init_debugfs(void)
4784{
4785	struct dentry *d_tracer;
4786
4787	d_tracer = tracing_init_dentry();
4788	if (!d_tracer)
4789		return 0;
4790
4791	ftrace_init_dyn_debugfs(d_tracer);
4792
4793	trace_create_file("set_ftrace_pid", 0644, d_tracer,
4794			    NULL, &ftrace_pid_fops);
 
 
 
 
 
4795
4796	ftrace_profile_debugfs(d_tracer);
 
 
 
 
4797
4798	return 0;
 
4799}
4800fs_initcall(ftrace_init_debugfs);
4801
4802/**
4803 * ftrace_kill - kill ftrace
4804 *
4805 * This function should be used by panic code. It stops ftrace
4806 * but in a not so nice way. If you need to simply kill ftrace
4807 * from a non-atomic section, use ftrace_kill.
4808 */
4809void ftrace_kill(void)
4810{
4811	ftrace_disabled = 1;
4812	ftrace_enabled = 0;
4813	clear_ftrace_function();
4814}
4815
4816/**
4817 * Test if ftrace is dead or not.
 
 
4818 */
4819int ftrace_is_dead(void)
4820{
4821	return ftrace_disabled;
4822}
4823
4824/**
4825 * register_ftrace_function - register a function for profiling
4826 * @ops - ops structure that holds the function for profiling.
 
 
 
4827 *
4828 * Register a function to be called by all functions in the
4829 * kernel.
4830 *
4831 * Note: @ops->func and all the functions it calls must be labeled
4832 *       with "notrace", otherwise it will go into a
4833 *       recursive loop.
4834 */
4835int register_ftrace_function(struct ftrace_ops *ops)
4836{
4837	int ret = -1;
4838
4839	ftrace_ops_init(ops);
 
4840
4841	mutex_lock(&ftrace_lock);
4842
4843	ret = ftrace_startup(ops, 0);
4844
4845	mutex_unlock(&ftrace_lock);
4846
4847	return ret;
4848}
4849EXPORT_SYMBOL_GPL(register_ftrace_function);
 
 
 
4850
4851/**
4852 * unregister_ftrace_function - unregister a function for profiling.
4853 * @ops - ops structure that holds the function to unregister
4854 *
4855 * Unregister a function that was added to be called by ftrace profiling.
4856 */
4857int unregister_ftrace_function(struct ftrace_ops *ops)
4858{
4859	int ret;
 
4860
4861	mutex_lock(&ftrace_lock);
4862	ret = ftrace_shutdown(ops, 0);
4863	mutex_unlock(&ftrace_lock);
 
 
 
 
 
 
 
4864
4865	return ret;
4866}
4867EXPORT_SYMBOL_GPL(unregister_ftrace_function);
4868
4869int
4870ftrace_enable_sysctl(struct ctl_table *table, int write,
4871		     void __user *buffer, size_t *lenp,
4872		     loff_t *ppos)
 
 
4873{
4874	int ret = -ENODEV;
4875
4876	mutex_lock(&ftrace_lock);
4877
4878	if (unlikely(ftrace_disabled))
4879		goto out;
4880
4881	ret = proc_dointvec(table, write, buffer, lenp, ppos);
4882
4883	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
4884		goto out;
4885
4886	last_ftrace_enabled = !!ftrace_enabled;
4887
4888	if (ftrace_enabled) {
 
4889
4890		ftrace_startup_sysctl();
4891
4892		/* we are starting ftrace again */
4893		if (ftrace_ops_list != &ftrace_list_end)
4894			update_ftrace_function();
 
 
 
4895
4896	} else {
4897		/* stopping ftrace calls (just send to ftrace_stub) */
4898		ftrace_trace_function = ftrace_stub;
 
 
 
 
 
 
 
4899
4900		ftrace_shutdown_sysctl();
 
 
 
4901	}
4902
4903 out:
4904	mutex_unlock(&ftrace_lock);
4905	return ret;
4906}
4907
4908#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 
4909
4910static int ftrace_graph_active;
4911static struct notifier_block ftrace_suspend_notifier;
4912
4913int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
4914{
4915	return 0;
4916}
4917
4918/* The callbacks that hook a function */
4919trace_func_graph_ret_t ftrace_graph_return =
4920			(trace_func_graph_ret_t)ftrace_stub;
4921trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
4922static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
4923
4924/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
4925static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
 
 
 
 
 
 
 
4926{
4927	int i;
4928	int ret = 0;
4929	unsigned long flags;
4930	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
4931	struct task_struct *g, *t;
4932
4933	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
4934		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
4935					* sizeof(struct ftrace_ret_stack),
4936					GFP_KERNEL);
4937		if (!ret_stack_list[i]) {
4938			start = 0;
4939			end = i;
4940			ret = -ENOMEM;
4941			goto free;
4942		}
4943	}
4944
4945	read_lock_irqsave(&tasklist_lock, flags);
4946	do_each_thread(g, t) {
4947		if (start == end) {
4948			ret = -EAGAIN;
4949			goto unlock;
4950		}
4951
4952		if (t->ret_stack == NULL) {
4953			atomic_set(&t->tracing_graph_pause, 0);
4954			atomic_set(&t->trace_overrun, 0);
4955			t->curr_ret_stack = -1;
4956			/* Make sure the tasks see the -1 first: */
4957			smp_wmb();
4958			t->ret_stack = ret_stack_list[start++];
4959		}
4960	} while_each_thread(g, t);
4961
4962unlock:
4963	read_unlock_irqrestore(&tasklist_lock, flags);
4964free:
4965	for (i = start; i < end; i++)
4966		kfree(ret_stack_list[i]);
4967	return ret;
4968}
4969
4970static void
4971ftrace_graph_probe_sched_switch(void *ignore,
4972			struct task_struct *prev, struct task_struct *next)
 
 
 
 
 
 
 
 
 
4973{
4974	unsigned long long timestamp;
4975	int index;
4976
4977	/*
4978	 * Does the user want to count the time a function was asleep.
4979	 * If so, do not update the time stamps.
4980	 */
4981	if (trace_flags & TRACE_ITER_SLEEP_TIME)
4982		return;
4983
4984	timestamp = trace_clock_local();
4985
4986	prev->ftrace_timestamp = timestamp;
 
 
 
 
4987
4988	/* only process tasks that we timestamped */
4989	if (!next->ftrace_timestamp)
4990		return;
 
 
 
 
 
 
4991
4992	/*
4993	 * Update all the counters in next to make up for the
4994	 * time next was sleeping.
4995	 */
4996	timestamp -= next->ftrace_timestamp;
4997
4998	for (index = next->curr_ret_stack; index >= 0; index--)
4999		next->ret_stack[index].calltime += timestamp;
5000}
 
5001
5002/* Allocate a return stack for each task */
5003static int start_graph_tracing(void)
5004{
5005	struct ftrace_ret_stack **ret_stack_list;
5006	int ret, cpu;
5007
5008	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
5009				sizeof(struct ftrace_ret_stack *),
5010				GFP_KERNEL);
5011
5012	if (!ret_stack_list)
5013		return -ENOMEM;
 
 
 
 
5014
5015	/* The cpu_boot init_task->ret_stack will never be freed */
5016	for_each_online_cpu(cpu) {
5017		if (!idle_task(cpu)->ret_stack)
5018			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
5019	}
 
 
 
 
 
5020
5021	do {
5022		ret = alloc_retstack_tasklist(ret_stack_list);
5023	} while (ret == -EAGAIN);
5024
5025	if (!ret) {
5026		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
5027		if (ret)
5028			pr_info("ftrace_graph: Couldn't activate tracepoint"
5029				" probe to kernel_sched_switch\n");
5030	}
5031
5032	kfree(ret_stack_list);
5033	return ret;
 
 
 
 
5034}
5035
5036/*
5037 * Hibernation protection.
5038 * The state of the current task is too much unstable during
5039 * suspend/restore to disk. We want to protect against that.
 
 
 
 
 
 
 
 
 
 
 
5040 */
5041static int
5042ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
5043							void *unused)
5044{
5045	switch (state) {
5046	case PM_HIBERNATION_PREPARE:
5047		pause_graph_tracing();
5048		break;
5049
5050	case PM_POST_HIBERNATION:
5051		unpause_graph_tracing();
5052		break;
5053	}
5054	return NOTIFY_DONE;
5055}
5056
5057/* Just a place holder for function graph */
5058static struct ftrace_ops fgraph_ops __read_mostly = {
5059	.func		= ftrace_stub,
5060	.flags		= FTRACE_OPS_FL_STUB | FTRACE_OPS_FL_GLOBAL |
5061				FTRACE_OPS_FL_RECURSION_SAFE,
5062};
5063
5064static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
5065{
5066	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
5067		return 0;
5068	return __ftrace_graph_entry(trace);
 
5069}
5070
5071/*
5072 * The function graph tracer should only trace the functions defined
5073 * by set_ftrace_filter and set_ftrace_notrace. If another function
5074 * tracer ops is registered, the graph tracer requires testing the
5075 * function against the global ops, and not just trace any function
5076 * that any ftrace_ops registered.
5077 */
5078static void update_function_graph_func(void)
5079{
5080	if (ftrace_ops_list == &ftrace_list_end ||
5081	    (ftrace_ops_list == &global_ops &&
5082	     global_ops.next == &ftrace_list_end))
5083		ftrace_graph_entry = __ftrace_graph_entry;
5084	else
5085		ftrace_graph_entry = ftrace_graph_entry_test;
5086}
5087
5088int register_ftrace_graph(trace_func_graph_ret_t retfunc,
5089			trace_func_graph_ent_t entryfunc)
5090{
5091	int ret = 0;
5092
5093	mutex_lock(&ftrace_lock);
 
5094
5095	/* we currently allow only one tracer registered at a time */
5096	if (ftrace_graph_active) {
5097		ret = -EBUSY;
5098		goto out;
 
 
 
 
5099	}
 
5100
5101	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
5102	register_pm_notifier(&ftrace_suspend_notifier);
 
5103
5104	ftrace_graph_active++;
5105	ret = start_graph_tracing();
5106	if (ret) {
5107		ftrace_graph_active--;
5108		goto out;
5109	}
5110
5111	ftrace_graph_return = retfunc;
 
 
 
 
 
 
 
 
 
 
 
5112
5113	/*
5114	 * Update the indirect function to the entryfunc, and the
5115	 * function that gets called to the entry_test first. Then
5116	 * call the update fgraph entry function to determine if
5117	 * the entryfunc should be called directly or not.
5118	 */
5119	__ftrace_graph_entry = entryfunc;
5120	ftrace_graph_entry = ftrace_graph_entry_test;
5121	update_function_graph_func();
5122
5123	ret = ftrace_startup(&fgraph_ops, FTRACE_START_FUNC_RET);
 
 
 
5124
5125out:
5126	mutex_unlock(&ftrace_lock);
5127	return ret;
5128}
5129
5130void unregister_ftrace_graph(void)
 
 
5131{
 
 
5132	mutex_lock(&ftrace_lock);
5133
5134	if (unlikely(!ftrace_graph_active))
5135		goto out;
5136
5137	ftrace_graph_active--;
5138	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
5139	ftrace_graph_entry = ftrace_graph_entry_stub;
5140	__ftrace_graph_entry = ftrace_graph_entry_stub;
5141	ftrace_shutdown(&fgraph_ops, FTRACE_STOP_FUNC_RET);
5142	unregister_pm_notifier(&ftrace_suspend_notifier);
5143	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
5144
5145 out:
5146	mutex_unlock(&ftrace_lock);
5147}
5148
5149static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
 
5150
5151static void
5152graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
5153{
5154	atomic_set(&t->tracing_graph_pause, 0);
5155	atomic_set(&t->trace_overrun, 0);
5156	t->ftrace_timestamp = 0;
5157	/* make curr_ret_stack visible before we add the ret_stack */
5158	smp_wmb();
5159	t->ret_stack = ret_stack;
5160}
5161
5162/*
5163 * Allocate a return stack for the idle task. May be the first
5164 * time through, or it may be done by CPU hotplug online.
5165 */
5166void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
5167{
5168	t->curr_ret_stack = -1;
5169	/*
5170	 * The idle task has no parent, it either has its own
5171	 * stack or no stack at all.
5172	 */
5173	if (t->ret_stack)
5174		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
5175
5176	if (ftrace_graph_active) {
5177		struct ftrace_ret_stack *ret_stack;
5178
5179		ret_stack = per_cpu(idle_ret_stack, cpu);
5180		if (!ret_stack) {
5181			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
5182					    * sizeof(struct ftrace_ret_stack),
5183					    GFP_KERNEL);
5184			if (!ret_stack)
5185				return;
5186			per_cpu(idle_ret_stack, cpu) = ret_stack;
5187		}
5188		graph_init_task(t, ret_stack);
5189	}
5190}
5191
5192/* Allocate a return stack for newly created task */
5193void ftrace_graph_init_task(struct task_struct *t)
5194{
5195	/* Make sure we do not use the parent ret_stack */
5196	t->ret_stack = NULL;
5197	t->curr_ret_stack = -1;
5198
5199	if (ftrace_graph_active) {
5200		struct ftrace_ret_stack *ret_stack;
5201
5202		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
5203				* sizeof(struct ftrace_ret_stack),
5204				GFP_KERNEL);
5205		if (!ret_stack)
5206			return;
5207		graph_init_task(t, ret_stack);
5208	}
5209}
5210
5211void ftrace_graph_exit_task(struct task_struct *t)
5212{
5213	struct ftrace_ret_stack	*ret_stack = t->ret_stack;
5214
5215	t->ret_stack = NULL;
5216	/* NULL must become visible to IRQs before we free it: */
5217	barrier();
5218
5219	kfree(ret_stack);
5220}
5221
5222void ftrace_graph_stop(void)
 
 
 
 
 
 
 
 
 
 
 
5223{
5224	ftrace_stop();
 
5225}
 
5226#endif