1/*
   2 * Infrastructure for profiling code inserted by 'gcc -pg'.
   3 *
   4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6 *
   7 * Originally ported from the -rt patch by:
   8 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9 *
  10 * Based on code in the latency_tracer, that is:
  11 *
  12 *  Copyright (C) 2004-2006 Ingo Molnar
  13 *  Copyright (C) 2004 Nadia Yvette Chambers
  14 */
  15
  16#include <linux/stop_machine.h>
  17#include <linux/clocksource.h>
  18#include <linux/sched/task.h>
  19#include <linux/kallsyms.h>
 
  20#include <linux/seq_file.h>
  21#include <linux/suspend.h>
  22#include <linux/tracefs.h>
  23#include <linux/hardirq.h>
  24#include <linux/kthread.h>
  25#include <linux/uaccess.h>
  26#include <linux/bsearch.h>
  27#include <linux/module.h>
  28#include <linux/ftrace.h>
  29#include <linux/sysctl.h>
  30#include <linux/slab.h>
  31#include <linux/ctype.h>
  32#include <linux/sort.h>
  33#include <linux/list.h>
  34#include <linux/hash.h>
  35#include <linux/rcupdate.h>
 
  36
  37#include <trace/events/sched.h>
  38
  39#include <asm/sections.h>
  40#include <asm/setup.h>
  41
 
  42#include "trace_output.h"
  43#include "trace_stat.h"
  44
  45#define FTRACE_WARN_ON(cond)			\
  46	({					\
  47		int ___r = cond;		\
  48		if (WARN_ON(___r))		\
  49			ftrace_kill();		\
  50		___r;				\
  51	})
  52
  53#define FTRACE_WARN_ON_ONCE(cond)		\
  54	({					\
  55		int ___r = cond;		\
  56		if (WARN_ON_ONCE(___r))		\
  57			ftrace_kill();		\
  58		___r;				\
  59	})
  60
  61/* hash bits for specific function selection */
  62#define FTRACE_HASH_BITS 7
  63#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  64#define FTRACE_HASH_DEFAULT_BITS 10
  65#define FTRACE_HASH_MAX_BITS 12
  66
  67#ifdef CONFIG_DYNAMIC_FTRACE
  68#define INIT_OPS_HASH(opsname)	\
  69	.func_hash		= &opsname.local_hash,			\
  70	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
  71#define ASSIGN_OPS_HASH(opsname, val) \
  72	.func_hash		= val, \
  73	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
  74#else
  75#define INIT_OPS_HASH(opsname)
  76#define ASSIGN_OPS_HASH(opsname, val)
  77#endif
  78
  79static struct ftrace_ops ftrace_list_end __read_mostly = {
 
 
 
 
 
  80	.func		= ftrace_stub,
  81	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
  82	INIT_OPS_HASH(ftrace_list_end)
  83};
  84
  85/* ftrace_enabled is a method to turn ftrace on or off */
  86int ftrace_enabled __read_mostly;
  87static int last_ftrace_enabled;
  88
  89/* Current function tracing op */
  90struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
  91/* What to set function_trace_op to */
  92static struct ftrace_ops *set_function_trace_op;
  93
  94static bool ftrace_pids_enabled(struct ftrace_ops *ops)
  95{
  96	struct trace_array *tr;
  97
  98	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
  99		return false;
 100
 101	tr = ops->private;
 102
 103	return tr->function_pids != NULL;
 104}
 105
 106static void ftrace_update_trampoline(struct ftrace_ops *ops);
 107
 108/*
 109 * ftrace_disabled is set when an anomaly is discovered.
 110 * ftrace_disabled is much stronger than ftrace_enabled.
 111 */
 112static int ftrace_disabled __read_mostly;
 113
 114static DEFINE_MUTEX(ftrace_lock);
 115
 116static struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
 117ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
 118static struct ftrace_ops global_ops;
 119
 120#if ARCH_SUPPORTS_FTRACE_OPS
 121static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
 122				 struct ftrace_ops *op, struct pt_regs *regs);
 123#else
 124/* See comment below, where ftrace_ops_list_func is defined */
 125static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
 126#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
 127#endif
 128
 129/*
 130 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
 131 * can use rcu_dereference_raw_notrace() is that elements removed from this list
 132 * are simply leaked, so there is no need to interact with a grace-period
 133 * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
 134 * concurrent insertions into the ftrace_global_list.
 135 *
 136 * Silly Alpha and silly pointer-speculation compiler optimizations!
 137 */
 138#define do_for_each_ftrace_op(op, list)			\
 139	op = rcu_dereference_raw_notrace(list);			\
 140	do
 141
 142/*
 143 * Optimized for just a single item in the list (as that is the normal case).
 144 */
 145#define while_for_each_ftrace_op(op)				\
 146	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
 147	       unlikely((op) != &ftrace_list_end))
 148
 149static inline void ftrace_ops_init(struct ftrace_ops *ops)
 150{
 151#ifdef CONFIG_DYNAMIC_FTRACE
 152	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
 153		mutex_init(&ops->local_hash.regex_lock);
 154		ops->func_hash = &ops->local_hash;
 155		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
 156	}
 157#endif
 158}
 159
 160/**
 161 * ftrace_nr_registered_ops - return number of ops registered
 162 *
 163 * Returns the number of ftrace_ops registered and tracing functions
 164 */
 165int ftrace_nr_registered_ops(void)
 166{
 167	struct ftrace_ops *ops;
 168	int cnt = 0;
 169
 170	mutex_lock(&ftrace_lock);
 171
 172	for (ops = rcu_dereference_protected(ftrace_ops_list,
 173					     lockdep_is_held(&ftrace_lock));
 174	     ops != &ftrace_list_end;
 175	     ops = rcu_dereference_protected(ops->next,
 176					     lockdep_is_held(&ftrace_lock)))
 177		cnt++;
 178
 179	mutex_unlock(&ftrace_lock);
 180
 181	return cnt;
 182}
 183
 184static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
 185			    struct ftrace_ops *op, struct pt_regs *regs)
 186{
 187	struct trace_array *tr = op->private;
 
 188
 189	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
 190		return;
 
 
 
 
 
 
 191
 192	op->saved_func(ip, parent_ip, op, regs);
 193}
 194
 195/**
 196 * clear_ftrace_function - reset the ftrace function
 197 *
 198 * This NULLs the ftrace function and in essence stops
 199 * tracing.  There may be lag
 200 */
 201void clear_ftrace_function(void)
 202{
 203	ftrace_trace_function = ftrace_stub;
 204}
 205
 206static void ftrace_sync(struct work_struct *work)
 207{
 208	/*
 209	 * This function is just a stub to implement a hard force
 210	 * of synchronize_sched(). This requires synchronizing
 211	 * tasks even in userspace and idle.
 212	 *
 213	 * Yes, function tracing is rude.
 214	 */
 215}
 216
 217static void ftrace_sync_ipi(void *data)
 218{
 219	/* Probably not needed, but do it anyway */
 220	smp_rmb();
 221}
 222
 223#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 224static void update_function_graph_func(void);
 225
 226/* Both enabled by default (can be cleared by function_graph tracer flags */
 227static bool fgraph_sleep_time = true;
 228static bool fgraph_graph_time = true;
 229
 230#else
 231static inline void update_function_graph_func(void) { }
 232#endif
 233
 234
 235static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
 236{
 237	/*
 238	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
 239	 * then it needs to call the list anyway.
 240	 */
 241	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
 242	    FTRACE_FORCE_LIST_FUNC)
 243		return ftrace_ops_list_func;
 244
 245	return ftrace_ops_get_func(ops);
 246}
 247
 248static void update_ftrace_function(void)
 249{
 250	ftrace_func_t func;
 251
 252	/*
 253	 * Prepare the ftrace_ops that the arch callback will use.
 254	 * If there's only one ftrace_ops registered, the ftrace_ops_list
 255	 * will point to the ops we want.
 256	 */
 257	set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
 258						lockdep_is_held(&ftrace_lock));
 259
 260	/* If there's no ftrace_ops registered, just call the stub function */
 261	if (set_function_trace_op == &ftrace_list_end) {
 262		func = ftrace_stub;
 263
 264	/*
 265	 * If we are at the end of the list and this ops is
 266	 * recursion safe and not dynamic and the arch supports passing ops,
 267	 * then have the mcount trampoline call the function directly.
 268	 */
 269	} else if (rcu_dereference_protected(ftrace_ops_list->next,
 270			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
 271		func = ftrace_ops_get_list_func(ftrace_ops_list);
 272
 273	} else {
 274		/* Just use the default ftrace_ops */
 275		set_function_trace_op = &ftrace_list_end;
 276		func = ftrace_ops_list_func;
 277	}
 278
 279	update_function_graph_func();
 280
 281	/* If there's no change, then do nothing more here */
 282	if (ftrace_trace_function == func)
 283		return;
 284
 285	/*
 286	 * If we are using the list function, it doesn't care
 287	 * about the function_trace_ops.
 288	 */
 289	if (func == ftrace_ops_list_func) {
 290		ftrace_trace_function = func;
 291		/*
 292		 * Don't even bother setting function_trace_ops,
 293		 * it would be racy to do so anyway.
 294		 */
 295		return;
 296	}
 297
 298#ifndef CONFIG_DYNAMIC_FTRACE
 299	/*
 300	 * For static tracing, we need to be a bit more careful.
 301	 * The function change takes affect immediately. Thus,
 302	 * we need to coorditate the setting of the function_trace_ops
 303	 * with the setting of the ftrace_trace_function.
 304	 *
 305	 * Set the function to the list ops, which will call the
 306	 * function we want, albeit indirectly, but it handles the
 307	 * ftrace_ops and doesn't depend on function_trace_op.
 308	 */
 309	ftrace_trace_function = ftrace_ops_list_func;
 310	/*
 311	 * Make sure all CPUs see this. Yes this is slow, but static
 312	 * tracing is slow and nasty to have enabled.
 313	 */
 314	schedule_on_each_cpu(ftrace_sync);
 315	/* Now all cpus are using the list ops. */
 316	function_trace_op = set_function_trace_op;
 317	/* Make sure the function_trace_op is visible on all CPUs */
 318	smp_wmb();
 319	/* Nasty way to force a rmb on all cpus */
 320	smp_call_function(ftrace_sync_ipi, NULL, 1);
 321	/* OK, we are all set to update the ftrace_trace_function now! */
 322#endif /* !CONFIG_DYNAMIC_FTRACE */
 323
 324	ftrace_trace_function = func;
 325}
 326
 327int using_ftrace_ops_list_func(void)
 328{
 329	return ftrace_trace_function == ftrace_ops_list_func;
 330}
 331
 332static void add_ftrace_ops(struct ftrace_ops __rcu **list,
 333			   struct ftrace_ops *ops)
 334{
 335	rcu_assign_pointer(ops->next, *list);
 336
 337	/*
 338	 * We are entering ops into the list but another
 339	 * CPU might be walking that list. We need to make sure
 340	 * the ops->next pointer is valid before another CPU sees
 341	 * the ops pointer included into the list.
 342	 */
 343	rcu_assign_pointer(*list, ops);
 344}
 345
 346static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
 347			     struct ftrace_ops *ops)
 348{
 349	struct ftrace_ops **p;
 350
 351	/*
 352	 * If we are removing the last function, then simply point
 353	 * to the ftrace_stub.
 354	 */
 355	if (rcu_dereference_protected(*list,
 356			lockdep_is_held(&ftrace_lock)) == ops &&
 357	    rcu_dereference_protected(ops->next,
 358			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
 359		*list = &ftrace_list_end;
 360		return 0;
 361	}
 362
 363	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
 364		if (*p == ops)
 365			break;
 366
 367	if (*p != ops)
 368		return -1;
 369
 370	*p = (*p)->next;
 371	return 0;
 372}
 373
 374static void ftrace_update_trampoline(struct ftrace_ops *ops);
 375
 376static int __register_ftrace_function(struct ftrace_ops *ops)
 377{
 378	if (ops->flags & FTRACE_OPS_FL_DELETED)
 379		return -EINVAL;
 380
 381	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
 382		return -EBUSY;
 383
 384#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 385	/*
 386	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
 387	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
 388	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
 389	 */
 390	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
 391	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
 392		return -EINVAL;
 393
 394	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
 395		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
 396#endif
 
 
 397
 398	if (!core_kernel_data((unsigned long)ops))
 399		ops->flags |= FTRACE_OPS_FL_DYNAMIC;
 400
 401	add_ftrace_ops(&ftrace_ops_list, ops);
 402
 403	/* Always save the function, and reset at unregistering */
 404	ops->saved_func = ops->func;
 405
 406	if (ftrace_pids_enabled(ops))
 407		ops->func = ftrace_pid_func;
 408
 409	ftrace_update_trampoline(ops);
 410
 411	if (ftrace_enabled)
 412		update_ftrace_function();
 413
 414	return 0;
 415}
 416
 417static int __unregister_ftrace_function(struct ftrace_ops *ops)
 418{
 419	int ret;
 420
 421	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
 422		return -EBUSY;
 423
 424	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
 425
 426	if (ret < 0)
 427		return ret;
 428
 429	if (ftrace_enabled)
 430		update_ftrace_function();
 431
 432	ops->func = ops->saved_func;
 433
 434	return 0;
 435}
 436
 437static void ftrace_update_pid_func(void)
 438{
 439	struct ftrace_ops *op;
 440
 441	/* Only do something if we are tracing something */
 442	if (ftrace_trace_function == ftrace_stub)
 443		return;
 444
 445	do_for_each_ftrace_op(op, ftrace_ops_list) {
 446		if (op->flags & FTRACE_OPS_FL_PID) {
 447			op->func = ftrace_pids_enabled(op) ?
 448				ftrace_pid_func : op->saved_func;
 449			ftrace_update_trampoline(op);
 450		}
 451	} while_for_each_ftrace_op(op);
 452
 453	update_ftrace_function();
 454}
 455
 456#ifdef CONFIG_FUNCTION_PROFILER
 457struct ftrace_profile {
 458	struct hlist_node		node;
 459	unsigned long			ip;
 460	unsigned long			counter;
 461#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 462	unsigned long long		time;
 463	unsigned long long		time_squared;
 464#endif
 465};
 466
 467struct ftrace_profile_page {
 468	struct ftrace_profile_page	*next;
 469	unsigned long			index;
 470	struct ftrace_profile		records[];
 471};
 472
 473struct ftrace_profile_stat {
 474	atomic_t			disabled;
 475	struct hlist_head		*hash;
 476	struct ftrace_profile_page	*pages;
 477	struct ftrace_profile_page	*start;
 478	struct tracer_stat		stat;
 479};
 480
 481#define PROFILE_RECORDS_SIZE						\
 482	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 483
 484#define PROFILES_PER_PAGE					\
 485	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 486
 487static int ftrace_profile_enabled __read_mostly;
 488
 489/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 490static DEFINE_MUTEX(ftrace_profile_lock);
 491
 492static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 493
 494#define FTRACE_PROFILE_HASH_BITS 10
 495#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
 496
 497static void *
 498function_stat_next(void *v, int idx)
 499{
 500	struct ftrace_profile *rec = v;
 501	struct ftrace_profile_page *pg;
 502
 503	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 504
 505 again:
 506	if (idx != 0)
 507		rec++;
 508
 509	if ((void *)rec >= (void *)&pg->records[pg->index]) {
 510		pg = pg->next;
 511		if (!pg)
 512			return NULL;
 513		rec = &pg->records[0];
 514		if (!rec->counter)
 515			goto again;
 516	}
 517
 518	return rec;
 519}
 520
 521static void *function_stat_start(struct tracer_stat *trace)
 522{
 523	struct ftrace_profile_stat *stat =
 524		container_of(trace, struct ftrace_profile_stat, stat);
 525
 526	if (!stat || !stat->start)
 527		return NULL;
 528
 529	return function_stat_next(&stat->start->records[0], 0);
 530}
 531
 532#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 533/* function graph compares on total time */
 534static int function_stat_cmp(void *p1, void *p2)
 535{
 536	struct ftrace_profile *a = p1;
 537	struct ftrace_profile *b = p2;
 538
 539	if (a->time < b->time)
 540		return -1;
 541	if (a->time > b->time)
 542		return 1;
 543	else
 544		return 0;
 545}
 546#else
 547/* not function graph compares against hits */
 548static int function_stat_cmp(void *p1, void *p2)
 549{
 550	struct ftrace_profile *a = p1;
 551	struct ftrace_profile *b = p2;
 552
 553	if (a->counter < b->counter)
 554		return -1;
 555	if (a->counter > b->counter)
 556		return 1;
 557	else
 558		return 0;
 559}
 560#endif
 561
 562static int function_stat_headers(struct seq_file *m)
 563{
 564#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 565	seq_puts(m, "  Function                               "
 566		 "Hit    Time            Avg             s^2\n"
 567		    "  --------                               "
 568		 "---    ----            ---             ---\n");
 569#else
 570	seq_puts(m, "  Function                               Hit\n"
 571		    "  --------                               ---\n");
 572#endif
 573	return 0;
 574}
 575
 576static int function_stat_show(struct seq_file *m, void *v)
 577{
 578	struct ftrace_profile *rec = v;
 579	char str[KSYM_SYMBOL_LEN];
 580	int ret = 0;
 581#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 582	static struct trace_seq s;
 583	unsigned long long avg;
 584	unsigned long long stddev;
 585#endif
 586	mutex_lock(&ftrace_profile_lock);
 587
 588	/* we raced with function_profile_reset() */
 589	if (unlikely(rec->counter == 0)) {
 590		ret = -EBUSY;
 591		goto out;
 592	}
 593
 594#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 595	avg = rec->time;
 596	do_div(avg, rec->counter);
 597	if (tracing_thresh && (avg < tracing_thresh))
 598		goto out;
 599#endif
 600
 601	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 602	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 603
 604#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 605	seq_puts(m, "    ");
 606
 607	/* Sample standard deviation (s^2) */
 608	if (rec->counter <= 1)
 609		stddev = 0;
 610	else {
 611		/*
 612		 * Apply Welford's method:
 613		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
 614		 */
 615		stddev = rec->counter * rec->time_squared -
 616			 rec->time * rec->time;
 617
 618		/*
 619		 * Divide only 1000 for ns^2 -> us^2 conversion.
 620		 * trace_print_graph_duration will divide 1000 again.
 621		 */
 622		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
 
 623	}
 624
 625	trace_seq_init(&s);
 626	trace_print_graph_duration(rec->time, &s);
 627	trace_seq_puts(&s, "    ");
 628	trace_print_graph_duration(avg, &s);
 629	trace_seq_puts(&s, "    ");
 630	trace_print_graph_duration(stddev, &s);
 631	trace_print_seq(m, &s);
 632#endif
 633	seq_putc(m, '\n');
 634out:
 635	mutex_unlock(&ftrace_profile_lock);
 636
 637	return ret;
 638}
 639
 640static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 641{
 642	struct ftrace_profile_page *pg;
 643
 644	pg = stat->pages = stat->start;
 645
 646	while (pg) {
 647		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 648		pg->index = 0;
 649		pg = pg->next;
 650	}
 651
 652	memset(stat->hash, 0,
 653	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 654}
 655
 656int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 657{
 658	struct ftrace_profile_page *pg;
 659	int functions;
 660	int pages;
 661	int i;
 662
 663	/* If we already allocated, do nothing */
 664	if (stat->pages)
 665		return 0;
 666
 667	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 668	if (!stat->pages)
 669		return -ENOMEM;
 670
 671#ifdef CONFIG_DYNAMIC_FTRACE
 672	functions = ftrace_update_tot_cnt;
 673#else
 674	/*
 675	 * We do not know the number of functions that exist because
 676	 * dynamic tracing is what counts them. With past experience
 677	 * we have around 20K functions. That should be more than enough.
 678	 * It is highly unlikely we will execute every function in
 679	 * the kernel.
 680	 */
 681	functions = 20000;
 682#endif
 683
 684	pg = stat->start = stat->pages;
 685
 686	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 687
 688	for (i = 1; i < pages; i++) {
 689		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 690		if (!pg->next)
 691			goto out_free;
 692		pg = pg->next;
 693	}
 694
 695	return 0;
 696
 697 out_free:
 698	pg = stat->start;
 699	while (pg) {
 700		unsigned long tmp = (unsigned long)pg;
 701
 702		pg = pg->next;
 703		free_page(tmp);
 704	}
 705
 706	stat->pages = NULL;
 707	stat->start = NULL;
 708
 709	return -ENOMEM;
 710}
 711
 712static int ftrace_profile_init_cpu(int cpu)
 713{
 714	struct ftrace_profile_stat *stat;
 715	int size;
 716
 717	stat = &per_cpu(ftrace_profile_stats, cpu);
 718
 719	if (stat->hash) {
 720		/* If the profile is already created, simply reset it */
 721		ftrace_profile_reset(stat);
 722		return 0;
 723	}
 724
 725	/*
 726	 * We are profiling all functions, but usually only a few thousand
 727	 * functions are hit. We'll make a hash of 1024 items.
 728	 */
 729	size = FTRACE_PROFILE_HASH_SIZE;
 730
 731	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 732
 733	if (!stat->hash)
 734		return -ENOMEM;
 735
 736	/* Preallocate the function profiling pages */
 737	if (ftrace_profile_pages_init(stat) < 0) {
 738		kfree(stat->hash);
 739		stat->hash = NULL;
 740		return -ENOMEM;
 741	}
 742
 743	return 0;
 744}
 745
 746static int ftrace_profile_init(void)
 747{
 748	int cpu;
 749	int ret = 0;
 750
 751	for_each_possible_cpu(cpu) {
 752		ret = ftrace_profile_init_cpu(cpu);
 753		if (ret)
 754			break;
 755	}
 756
 757	return ret;
 758}
 759
 760/* interrupts must be disabled */
 761static struct ftrace_profile *
 762ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 763{
 764	struct ftrace_profile *rec;
 765	struct hlist_head *hhd;
 766	unsigned long key;
 767
 768	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
 769	hhd = &stat->hash[key];
 770
 771	if (hlist_empty(hhd))
 772		return NULL;
 773
 774	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
 775		if (rec->ip == ip)
 776			return rec;
 777	}
 778
 779	return NULL;
 780}
 781
 782static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 783			       struct ftrace_profile *rec)
 784{
 785	unsigned long key;
 786
 787	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
 788	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 789}
 790
 791/*
 792 * The memory is already allocated, this simply finds a new record to use.
 793 */
 794static struct ftrace_profile *
 795ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 796{
 797	struct ftrace_profile *rec = NULL;
 798
 799	/* prevent recursion (from NMIs) */
 800	if (atomic_inc_return(&stat->disabled) != 1)
 801		goto out;
 802
 803	/*
 804	 * Try to find the function again since an NMI
 805	 * could have added it
 806	 */
 807	rec = ftrace_find_profiled_func(stat, ip);
 808	if (rec)
 809		goto out;
 810
 811	if (stat->pages->index == PROFILES_PER_PAGE) {
 812		if (!stat->pages->next)
 813			goto out;
 814		stat->pages = stat->pages->next;
 815	}
 816
 817	rec = &stat->pages->records[stat->pages->index++];
 818	rec->ip = ip;
 819	ftrace_add_profile(stat, rec);
 820
 821 out:
 822	atomic_dec(&stat->disabled);
 823
 824	return rec;
 825}
 826
 827static void
 828function_profile_call(unsigned long ip, unsigned long parent_ip,
 829		      struct ftrace_ops *ops, struct pt_regs *regs)
 830{
 831	struct ftrace_profile_stat *stat;
 832	struct ftrace_profile *rec;
 833	unsigned long flags;
 834
 835	if (!ftrace_profile_enabled)
 836		return;
 837
 838	local_irq_save(flags);
 839
 840	stat = this_cpu_ptr(&ftrace_profile_stats);
 841	if (!stat->hash || !ftrace_profile_enabled)
 842		goto out;
 843
 844	rec = ftrace_find_profiled_func(stat, ip);
 845	if (!rec) {
 846		rec = ftrace_profile_alloc(stat, ip);
 847		if (!rec)
 848			goto out;
 849	}
 850
 851	rec->counter++;
 852 out:
 853	local_irq_restore(flags);
 854}
 855
 856#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 
 
 
 
 
 
 
 857static int profile_graph_entry(struct ftrace_graph_ent *trace)
 858{
 859	int index = trace->depth;
 860
 861	function_profile_call(trace->func, 0, NULL, NULL);
 862
 863	/* If function graph is shutting down, ret_stack can be NULL */
 864	if (!current->ret_stack)
 865		return 0;
 866
 867	if (index >= 0 && index < FTRACE_RETFUNC_DEPTH)
 868		current->ret_stack[index].subtime = 0;
 
 869
 870	return 1;
 871}
 872
 873static void profile_graph_return(struct ftrace_graph_ret *trace)
 874{
 
 875	struct ftrace_profile_stat *stat;
 876	unsigned long long calltime;
 877	struct ftrace_profile *rec;
 878	unsigned long flags;
 879
 880	local_irq_save(flags);
 881	stat = this_cpu_ptr(&ftrace_profile_stats);
 882	if (!stat->hash || !ftrace_profile_enabled)
 883		goto out;
 884
 885	/* If the calltime was zero'd ignore it */
 886	if (!trace->calltime)
 887		goto out;
 888
 889	calltime = trace->rettime - trace->calltime;
 890
 891	if (!fgraph_graph_time) {
 892		int index;
 893
 894		index = trace->depth;
 895
 896		/* Append this call time to the parent time to subtract */
 897		if (index)
 898			current->ret_stack[index - 1].subtime += calltime;
 899
 900		if (current->ret_stack[index].subtime < calltime)
 901			calltime -= current->ret_stack[index].subtime;
 
 
 902		else
 903			calltime = 0;
 904	}
 905
 906	rec = ftrace_find_profiled_func(stat, trace->func);
 907	if (rec) {
 908		rec->time += calltime;
 909		rec->time_squared += calltime * calltime;
 910	}
 911
 912 out:
 913	local_irq_restore(flags);
 914}
 915
 
 
 
 
 
 916static int register_ftrace_profiler(void)
 917{
 918	return register_ftrace_graph(&profile_graph_return,
 919				     &profile_graph_entry);
 920}
 921
 922static void unregister_ftrace_profiler(void)
 923{
 924	unregister_ftrace_graph();
 925}
 926#else
 927static struct ftrace_ops ftrace_profile_ops __read_mostly = {
 928	.func		= function_profile_call,
 929	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
 930	INIT_OPS_HASH(ftrace_profile_ops)
 931};
 932
 933static int register_ftrace_profiler(void)
 934{
 935	return register_ftrace_function(&ftrace_profile_ops);
 936}
 937
 938static void unregister_ftrace_profiler(void)
 939{
 940	unregister_ftrace_function(&ftrace_profile_ops);
 941}
 942#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 943
 944static ssize_t
 945ftrace_profile_write(struct file *filp, const char __user *ubuf,
 946		     size_t cnt, loff_t *ppos)
 947{
 948	unsigned long val;
 949	int ret;
 950
 951	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 952	if (ret)
 953		return ret;
 954
 955	val = !!val;
 956
 957	mutex_lock(&ftrace_profile_lock);
 958	if (ftrace_profile_enabled ^ val) {
 959		if (val) {
 960			ret = ftrace_profile_init();
 961			if (ret < 0) {
 962				cnt = ret;
 963				goto out;
 964			}
 965
 966			ret = register_ftrace_profiler();
 967			if (ret < 0) {
 968				cnt = ret;
 969				goto out;
 970			}
 971			ftrace_profile_enabled = 1;
 972		} else {
 973			ftrace_profile_enabled = 0;
 974			/*
 975			 * unregister_ftrace_profiler calls stop_machine
 976			 * so this acts like an synchronize_sched.
 977			 */
 978			unregister_ftrace_profiler();
 979		}
 980	}
 981 out:
 982	mutex_unlock(&ftrace_profile_lock);
 983
 984	*ppos += cnt;
 985
 986	return cnt;
 987}
 988
 989static ssize_t
 990ftrace_profile_read(struct file *filp, char __user *ubuf,
 991		     size_t cnt, loff_t *ppos)
 992{
 993	char buf[64];		/* big enough to hold a number */
 994	int r;
 995
 996	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 997	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 998}
 999
1000static const struct file_operations ftrace_profile_fops = {
1001	.open		= tracing_open_generic,
1002	.read		= ftrace_profile_read,
1003	.write		= ftrace_profile_write,
1004	.llseek		= default_llseek,
1005};
1006
1007/* used to initialize the real stat files */
1008static struct tracer_stat function_stats __initdata = {
1009	.name		= "functions",
1010	.stat_start	= function_stat_start,
1011	.stat_next	= function_stat_next,
1012	.stat_cmp	= function_stat_cmp,
1013	.stat_headers	= function_stat_headers,
1014	.stat_show	= function_stat_show
1015};
1016
1017static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1018{
1019	struct ftrace_profile_stat *stat;
1020	struct dentry *entry;
1021	char *name;
1022	int ret;
1023	int cpu;
1024
1025	for_each_possible_cpu(cpu) {
1026		stat = &per_cpu(ftrace_profile_stats, cpu);
1027
1028		name = kasprintf(GFP_KERNEL, "function%d", cpu);
1029		if (!name) {
1030			/*
1031			 * The files created are permanent, if something happens
1032			 * we still do not free memory.
1033			 */
1034			WARN(1,
1035			     "Could not allocate stat file for cpu %d\n",
1036			     cpu);
1037			return;
1038		}
1039		stat->stat = function_stats;
1040		stat->stat.name = name;
1041		ret = register_stat_tracer(&stat->stat);
1042		if (ret) {
1043			WARN(1,
1044			     "Could not register function stat for cpu %d\n",
1045			     cpu);
1046			kfree(name);
1047			return;
1048		}
1049	}
1050
1051	entry = tracefs_create_file("function_profile_enabled", 0644,
1052				    d_tracer, NULL, &ftrace_profile_fops);
1053	if (!entry)
1054		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
1055}
1056
1057#else /* CONFIG_FUNCTION_PROFILER */
1058static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1059{
1060}
1061#endif /* CONFIG_FUNCTION_PROFILER */
1062
1063static struct pid * const ftrace_swapper_pid = &init_struct_pid;
1064
1065#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1066static int ftrace_graph_active;
1067#else
1068# define ftrace_graph_active 0
1069#endif
1070
1071#ifdef CONFIG_DYNAMIC_FTRACE
1072
1073static struct ftrace_ops *removed_ops;
1074
1075/*
1076 * Set when doing a global update, like enabling all recs or disabling them.
1077 * It is not set when just updating a single ftrace_ops.
1078 */
1079static bool update_all_ops;
1080
1081#ifndef CONFIG_FTRACE_MCOUNT_RECORD
1082# error Dynamic ftrace depends on MCOUNT_RECORD
1083#endif
1084
1085struct ftrace_func_entry {
1086	struct hlist_node hlist;
1087	unsigned long ip;
1088};
1089
1090struct ftrace_func_probe {
1091	struct ftrace_probe_ops	*probe_ops;
1092	struct ftrace_ops	ops;
1093	struct trace_array	*tr;
1094	struct list_head	list;
1095	void			*data;
1096	int			ref;
1097};
1098
1099/*
1100 * We make these constant because no one should touch them,
1101 * but they are used as the default "empty hash", to avoid allocating
1102 * it all the time. These are in a read only section such that if
1103 * anyone does try to modify it, it will cause an exception.
1104 */
1105static const struct hlist_head empty_buckets[1];
1106static const struct ftrace_hash empty_hash = {
1107	.buckets = (struct hlist_head *)empty_buckets,
1108};
1109#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1110
1111static struct ftrace_ops global_ops = {
1112	.func				= ftrace_stub,
1113	.local_hash.notrace_hash	= EMPTY_HASH,
1114	.local_hash.filter_hash		= EMPTY_HASH,
1115	INIT_OPS_HASH(global_ops)
1116	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1117					  FTRACE_OPS_FL_INITIALIZED |
1118					  FTRACE_OPS_FL_PID,
1119};
1120
1121/*
1122 * Used by the stack undwinder to know about dynamic ftrace trampolines.
1123 */
1124struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
1125{
1126	struct ftrace_ops *op = NULL;
1127
1128	/*
1129	 * Some of the ops may be dynamically allocated,
1130	 * they are freed after a synchronize_sched().
1131	 */
1132	preempt_disable_notrace();
1133
1134	do_for_each_ftrace_op(op, ftrace_ops_list) {
1135		/*
1136		 * This is to check for dynamically allocated trampolines.
1137		 * Trampolines that are in kernel text will have
1138		 * core_kernel_text() return true.
1139		 */
1140		if (op->trampoline && op->trampoline_size)
1141			if (addr >= op->trampoline &&
1142			    addr < op->trampoline + op->trampoline_size) {
1143				preempt_enable_notrace();
1144				return op;
1145			}
1146	} while_for_each_ftrace_op(op);
1147	preempt_enable_notrace();
1148
1149	return NULL;
1150}
1151
1152/*
1153 * This is used by __kernel_text_address() to return true if the
1154 * address is on a dynamically allocated trampoline that would
1155 * not return true for either core_kernel_text() or
1156 * is_module_text_address().
1157 */
1158bool is_ftrace_trampoline(unsigned long addr)
1159{
1160	return ftrace_ops_trampoline(addr) != NULL;
1161}
1162
1163struct ftrace_page {
1164	struct ftrace_page	*next;
1165	struct dyn_ftrace	*records;
1166	int			index;
1167	int			size;
1168};
1169
1170#define ENTRY_SIZE sizeof(struct dyn_ftrace)
1171#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1172
1173/* estimate from running different kernels */
1174#define NR_TO_INIT		10000
1175
1176static struct ftrace_page	*ftrace_pages_start;
1177static struct ftrace_page	*ftrace_pages;
1178
1179static __always_inline unsigned long
1180ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
1181{
1182	if (hash->size_bits > 0)
1183		return hash_long(ip, hash->size_bits);
1184
1185	return 0;
1186}
1187
1188/* Only use this function if ftrace_hash_empty() has already been tested */
1189static __always_inline struct ftrace_func_entry *
1190__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1191{
1192	unsigned long key;
1193	struct ftrace_func_entry *entry;
1194	struct hlist_head *hhd;
1195
1196	key = ftrace_hash_key(hash, ip);
1197	hhd = &hash->buckets[key];
1198
1199	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1200		if (entry->ip == ip)
1201			return entry;
1202	}
1203	return NULL;
1204}
1205
1206/**
1207 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
1208 * @hash: The hash to look at
1209 * @ip: The instruction pointer to test
1210 *
1211 * Search a given @hash to see if a given instruction pointer (@ip)
1212 * exists in it.
1213 *
1214 * Returns the entry that holds the @ip if found. NULL otherwise.
1215 */
1216struct ftrace_func_entry *
1217ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1218{
1219	if (ftrace_hash_empty(hash))
1220		return NULL;
1221
1222	return __ftrace_lookup_ip(hash, ip);
1223}
1224
1225static void __add_hash_entry(struct ftrace_hash *hash,
1226			     struct ftrace_func_entry *entry)
1227{
1228	struct hlist_head *hhd;
1229	unsigned long key;
1230
1231	key = ftrace_hash_key(hash, entry->ip);
1232	hhd = &hash->buckets[key];
1233	hlist_add_head(&entry->hlist, hhd);
1234	hash->count++;
1235}
1236
1237static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1238{
1239	struct ftrace_func_entry *entry;
1240
1241	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1242	if (!entry)
1243		return -ENOMEM;
1244
1245	entry->ip = ip;
1246	__add_hash_entry(hash, entry);
1247
1248	return 0;
1249}
1250
1251static void
1252free_hash_entry(struct ftrace_hash *hash,
1253		  struct ftrace_func_entry *entry)
1254{
1255	hlist_del(&entry->hlist);
1256	kfree(entry);
1257	hash->count--;
1258}
1259
1260static void
1261remove_hash_entry(struct ftrace_hash *hash,
1262		  struct ftrace_func_entry *entry)
1263{
1264	hlist_del_rcu(&entry->hlist);
1265	hash->count--;
1266}
1267
1268static void ftrace_hash_clear(struct ftrace_hash *hash)
1269{
1270	struct hlist_head *hhd;
1271	struct hlist_node *tn;
1272	struct ftrace_func_entry *entry;
1273	int size = 1 << hash->size_bits;
1274	int i;
1275
1276	if (!hash->count)
1277		return;
1278
1279	for (i = 0; i < size; i++) {
1280		hhd = &hash->buckets[i];
1281		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1282			free_hash_entry(hash, entry);
1283	}
1284	FTRACE_WARN_ON(hash->count);
1285}
1286
1287static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
1288{
1289	list_del(&ftrace_mod->list);
1290	kfree(ftrace_mod->module);
1291	kfree(ftrace_mod->func);
1292	kfree(ftrace_mod);
1293}
1294
1295static void clear_ftrace_mod_list(struct list_head *head)
1296{
1297	struct ftrace_mod_load *p, *n;
1298
1299	/* stack tracer isn't supported yet */
1300	if (!head)
1301		return;
1302
1303	mutex_lock(&ftrace_lock);
1304	list_for_each_entry_safe(p, n, head, list)
1305		free_ftrace_mod(p);
1306	mutex_unlock(&ftrace_lock);
1307}
1308
1309static void free_ftrace_hash(struct ftrace_hash *hash)
1310{
1311	if (!hash || hash == EMPTY_HASH)
1312		return;
1313	ftrace_hash_clear(hash);
1314	kfree(hash->buckets);
1315	kfree(hash);
1316}
1317
1318static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1319{
1320	struct ftrace_hash *hash;
1321
1322	hash = container_of(rcu, struct ftrace_hash, rcu);
1323	free_ftrace_hash(hash);
1324}
1325
1326static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1327{
1328	if (!hash || hash == EMPTY_HASH)
1329		return;
1330	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1331}
1332
1333void ftrace_free_filter(struct ftrace_ops *ops)
1334{
1335	ftrace_ops_init(ops);
1336	free_ftrace_hash(ops->func_hash->filter_hash);
1337	free_ftrace_hash(ops->func_hash->notrace_hash);
1338}
1339
1340static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1341{
1342	struct ftrace_hash *hash;
1343	int size;
1344
1345	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1346	if (!hash)
1347		return NULL;
1348
1349	size = 1 << size_bits;
1350	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1351
1352	if (!hash->buckets) {
1353		kfree(hash);
1354		return NULL;
1355	}
1356
1357	hash->size_bits = size_bits;
1358
1359	return hash;
1360}
1361
1362
1363static int ftrace_add_mod(struct trace_array *tr,
1364			  const char *func, const char *module,
1365			  int enable)
1366{
1367	struct ftrace_mod_load *ftrace_mod;
1368	struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;
1369
1370	ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
1371	if (!ftrace_mod)
1372		return -ENOMEM;
1373
1374	ftrace_mod->func = kstrdup(func, GFP_KERNEL);
1375	ftrace_mod->module = kstrdup(module, GFP_KERNEL);
1376	ftrace_mod->enable = enable;
1377
1378	if (!ftrace_mod->func || !ftrace_mod->module)
1379		goto out_free;
1380
1381	list_add(&ftrace_mod->list, mod_head);
1382
1383	return 0;
1384
1385 out_free:
1386	free_ftrace_mod(ftrace_mod);
1387
1388	return -ENOMEM;
1389}
1390
1391static struct ftrace_hash *
1392alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1393{
1394	struct ftrace_func_entry *entry;
1395	struct ftrace_hash *new_hash;
1396	int size;
1397	int ret;
1398	int i;
1399
1400	new_hash = alloc_ftrace_hash(size_bits);
1401	if (!new_hash)
1402		return NULL;
1403
1404	if (hash)
1405		new_hash->flags = hash->flags;
1406
1407	/* Empty hash? */
1408	if (ftrace_hash_empty(hash))
1409		return new_hash;
1410
1411	size = 1 << hash->size_bits;
1412	for (i = 0; i < size; i++) {
1413		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1414			ret = add_hash_entry(new_hash, entry->ip);
1415			if (ret < 0)
1416				goto free_hash;
1417		}
1418	}
1419
1420	FTRACE_WARN_ON(new_hash->count != hash->count);
1421
1422	return new_hash;
1423
1424 free_hash:
1425	free_ftrace_hash(new_hash);
1426	return NULL;
1427}
1428
1429static void
1430ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1431static void
1432ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1433
1434static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1435				       struct ftrace_hash *new_hash);
1436
1437static struct ftrace_hash *
1438__ftrace_hash_move(struct ftrace_hash *src)
1439{
1440	struct ftrace_func_entry *entry;
1441	struct hlist_node *tn;
1442	struct hlist_head *hhd;
1443	struct ftrace_hash *new_hash;
1444	int size = src->count;
 
1445	int bits = 0;
1446	int i;
1447
1448	/*
1449	 * If the new source is empty, just return the empty_hash.
1450	 */
1451	if (ftrace_hash_empty(src))
1452		return EMPTY_HASH;
1453
1454	/*
1455	 * Make the hash size about 1/2 the # found
1456	 */
1457	for (size /= 2; size; size >>= 1)
1458		bits++;
1459
1460	/* Don't allocate too much */
1461	if (bits > FTRACE_HASH_MAX_BITS)
1462		bits = FTRACE_HASH_MAX_BITS;
1463
1464	new_hash = alloc_ftrace_hash(bits);
1465	if (!new_hash)
1466		return NULL;
1467
1468	new_hash->flags = src->flags;
1469
1470	size = 1 << src->size_bits;
1471	for (i = 0; i < size; i++) {
1472		hhd = &src->buckets[i];
1473		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1474			remove_hash_entry(src, entry);
1475			__add_hash_entry(new_hash, entry);
1476		}
1477	}
1478
1479	return new_hash;
1480}
1481
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1482static int
1483ftrace_hash_move(struct ftrace_ops *ops, int enable,
1484		 struct ftrace_hash **dst, struct ftrace_hash *src)
1485{
1486	struct ftrace_hash *new_hash;
1487	int ret;
1488
1489	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
1490	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
1491		return -EINVAL;
1492
1493	new_hash = __ftrace_hash_move(src);
1494	if (!new_hash)
1495		return -ENOMEM;
1496
1497	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
1498	if (enable) {
1499		/* IPMODIFY should be updated only when filter_hash updating */
1500		ret = ftrace_hash_ipmodify_update(ops, new_hash);
1501		if (ret < 0) {
1502			free_ftrace_hash(new_hash);
1503			return ret;
1504		}
1505	}
1506
1507	/*
1508	 * Remove the current set, update the hash and add
1509	 * them back.
1510	 */
1511	ftrace_hash_rec_disable_modify(ops, enable);
1512
1513	rcu_assign_pointer(*dst, new_hash);
1514
1515	ftrace_hash_rec_enable_modify(ops, enable);
1516
1517	return 0;
1518}
1519
1520static bool hash_contains_ip(unsigned long ip,
1521			     struct ftrace_ops_hash *hash)
1522{
1523	/*
1524	 * The function record is a match if it exists in the filter
1525	 * hash and not in the notrace hash. Note, an emty hash is
1526	 * considered a match for the filter hash, but an empty
1527	 * notrace hash is considered not in the notrace hash.
1528	 */
1529	return (ftrace_hash_empty(hash->filter_hash) ||
1530		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1531		(ftrace_hash_empty(hash->notrace_hash) ||
1532		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1533}
1534
1535/*
1536 * Test the hashes for this ops to see if we want to call
1537 * the ops->func or not.
1538 *
1539 * It's a match if the ip is in the ops->filter_hash or
1540 * the filter_hash does not exist or is empty,
1541 *  AND
1542 * the ip is not in the ops->notrace_hash.
1543 *
1544 * This needs to be called with preemption disabled as
1545 * the hashes are freed with call_rcu_sched().
1546 */
1547static int
1548ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1549{
1550	struct ftrace_ops_hash hash;
1551	int ret;
1552
1553#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1554	/*
1555	 * There's a small race when adding ops that the ftrace handler
1556	 * that wants regs, may be called without them. We can not
1557	 * allow that handler to be called if regs is NULL.
1558	 */
1559	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1560		return 0;
1561#endif
1562
1563	rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
1564	rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
1565
1566	if (hash_contains_ip(ip, &hash))
1567		ret = 1;
1568	else
1569		ret = 0;
1570
1571	return ret;
1572}
1573
1574/*
1575 * This is a double for. Do not use 'break' to break out of the loop,
1576 * you must use a goto.
1577 */
1578#define do_for_each_ftrace_rec(pg, rec)					\
1579	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1580		int _____i;						\
1581		for (_____i = 0; _____i < pg->index; _____i++) {	\
1582			rec = &pg->records[_____i];
1583
1584#define while_for_each_ftrace_rec()		\
1585		}				\
1586	}
1587
1588
1589static int ftrace_cmp_recs(const void *a, const void *b)
1590{
1591	const struct dyn_ftrace *key = a;
1592	const struct dyn_ftrace *rec = b;
1593
1594	if (key->flags < rec->ip)
1595		return -1;
1596	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1597		return 1;
1598	return 0;
1599}
1600
1601/**
1602 * ftrace_location_range - return the first address of a traced location
1603 *	if it touches the given ip range
1604 * @start: start of range to search.
1605 * @end: end of range to search (inclusive). @end points to the last byte
1606 *	to check.
1607 *
1608 * Returns rec->ip if the related ftrace location is a least partly within
1609 * the given address range. That is, the first address of the instruction
1610 * that is either a NOP or call to the function tracer. It checks the ftrace
1611 * internal tables to determine if the address belongs or not.
1612 */
1613unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1614{
1615	struct ftrace_page *pg;
1616	struct dyn_ftrace *rec;
1617	struct dyn_ftrace key;
1618
1619	key.ip = start;
1620	key.flags = end;	/* overload flags, as it is unsigned long */
1621
1622	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1623		if (end < pg->records[0].ip ||
1624		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1625			continue;
1626		rec = bsearch(&key, pg->records, pg->index,
1627			      sizeof(struct dyn_ftrace),
1628			      ftrace_cmp_recs);
1629		if (rec)
1630			return rec->ip;
1631	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1632
1633	return 0;
1634}
1635
1636/**
1637 * ftrace_location - return true if the ip giving is a traced location
1638 * @ip: the instruction pointer to check
1639 *
1640 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1641 * That is, the instruction that is either a NOP or call to
1642 * the function tracer. It checks the ftrace internal tables to
1643 * determine if the address belongs or not.
1644 */
1645unsigned long ftrace_location(unsigned long ip)
1646{
1647	return ftrace_location_range(ip, ip);
1648}
1649
1650/**
1651 * ftrace_text_reserved - return true if range contains an ftrace location
1652 * @start: start of range to search
1653 * @end: end of range to search (inclusive). @end points to the last byte to check.
1654 *
1655 * Returns 1 if @start and @end contains a ftrace location.
1656 * That is, the instruction that is either a NOP or call to
1657 * the function tracer. It checks the ftrace internal tables to
1658 * determine if the address belongs or not.
1659 */
1660int ftrace_text_reserved(const void *start, const void *end)
1661{
1662	unsigned long ret;
1663
1664	ret = ftrace_location_range((unsigned long)start,
1665				    (unsigned long)end);
1666
1667	return (int)!!ret;
1668}
1669
1670/* Test if ops registered to this rec needs regs */
1671static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
1672{
1673	struct ftrace_ops *ops;
1674	bool keep_regs = false;
1675
1676	for (ops = ftrace_ops_list;
1677	     ops != &ftrace_list_end; ops = ops->next) {
1678		/* pass rec in as regs to have non-NULL val */
1679		if (ftrace_ops_test(ops, rec->ip, rec)) {
1680			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1681				keep_regs = true;
1682				break;
1683			}
1684		}
1685	}
1686
1687	return  keep_regs;
1688}
1689
 
 
 
 
 
1690static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1691				     int filter_hash,
1692				     bool inc)
1693{
1694	struct ftrace_hash *hash;
1695	struct ftrace_hash *other_hash;
1696	struct ftrace_page *pg;
1697	struct dyn_ftrace *rec;
1698	bool update = false;
1699	int count = 0;
1700	int all = false;
1701
1702	/* Only update if the ops has been registered */
1703	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1704		return false;
1705
1706	/*
1707	 * In the filter_hash case:
1708	 *   If the count is zero, we update all records.
1709	 *   Otherwise we just update the items in the hash.
1710	 *
1711	 * In the notrace_hash case:
1712	 *   We enable the update in the hash.
1713	 *   As disabling notrace means enabling the tracing,
1714	 *   and enabling notrace means disabling, the inc variable
1715	 *   gets inversed.
1716	 */
1717	if (filter_hash) {
1718		hash = ops->func_hash->filter_hash;
1719		other_hash = ops->func_hash->notrace_hash;
1720		if (ftrace_hash_empty(hash))
1721			all = true;
1722	} else {
1723		inc = !inc;
1724		hash = ops->func_hash->notrace_hash;
1725		other_hash = ops->func_hash->filter_hash;
1726		/*
1727		 * If the notrace hash has no items,
1728		 * then there's nothing to do.
1729		 */
1730		if (ftrace_hash_empty(hash))
1731			return false;
1732	}
1733
1734	do_for_each_ftrace_rec(pg, rec) {
1735		int in_other_hash = 0;
1736		int in_hash = 0;
1737		int match = 0;
1738
1739		if (rec->flags & FTRACE_FL_DISABLED)
1740			continue;
1741
1742		if (all) {
1743			/*
1744			 * Only the filter_hash affects all records.
1745			 * Update if the record is not in the notrace hash.
1746			 */
1747			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1748				match = 1;
1749		} else {
1750			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1751			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1752
1753			/*
1754			 * If filter_hash is set, we want to match all functions
1755			 * that are in the hash but not in the other hash.
1756			 *
1757			 * If filter_hash is not set, then we are decrementing.
1758			 * That means we match anything that is in the hash
1759			 * and also in the other_hash. That is, we need to turn
1760			 * off functions in the other hash because they are disabled
1761			 * by this hash.
1762			 */
1763			if (filter_hash && in_hash && !in_other_hash)
1764				match = 1;
1765			else if (!filter_hash && in_hash &&
1766				 (in_other_hash || ftrace_hash_empty(other_hash)))
1767				match = 1;
1768		}
1769		if (!match)
1770			continue;
1771
1772		if (inc) {
1773			rec->flags++;
1774			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1775				return false;
1776
 
 
 
1777			/*
1778			 * If there's only a single callback registered to a
1779			 * function, and the ops has a trampoline registered
1780			 * for it, then we can call it directly.
1781			 */
1782			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1783				rec->flags |= FTRACE_FL_TRAMP;
1784			else
1785				/*
1786				 * If we are adding another function callback
1787				 * to this function, and the previous had a
1788				 * custom trampoline in use, then we need to go
1789				 * back to the default trampoline.
1790				 */
1791				rec->flags &= ~FTRACE_FL_TRAMP;
1792
1793			/*
1794			 * If any ops wants regs saved for this function
1795			 * then all ops will get saved regs.
1796			 */
1797			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1798				rec->flags |= FTRACE_FL_REGS;
1799		} else {
1800			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1801				return false;
1802			rec->flags--;
1803
1804			/*
 
 
 
 
 
 
 
 
 
1805			 * If the rec had REGS enabled and the ops that is
1806			 * being removed had REGS set, then see if there is
1807			 * still any ops for this record that wants regs.
1808			 * If not, we can stop recording them.
1809			 */
1810			if (ftrace_rec_count(rec) > 0 &&
1811			    rec->flags & FTRACE_FL_REGS &&
1812			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1813				if (!test_rec_ops_needs_regs(rec))
1814					rec->flags &= ~FTRACE_FL_REGS;
1815			}
1816
1817			/*
1818			 * If the rec had TRAMP enabled, then it needs to
1819			 * be cleared. As TRAMP can only be enabled iff
1820			 * there is only a single ops attached to it.
1821			 * In otherwords, always disable it on decrementing.
1822			 * In the future, we may set it if rec count is
1823			 * decremented to one, and the ops that is left
1824			 * has a trampoline.
1825			 */
1826			rec->flags &= ~FTRACE_FL_TRAMP;
 
 
 
 
1827
1828			/*
1829			 * flags will be cleared in ftrace_check_record()
1830			 * if rec count is zero.
1831			 */
1832		}
1833		count++;
1834
1835		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
1836		update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;
1837
1838		/* Shortcut, if we handled all records, we are done. */
1839		if (!all && count == hash->count)
1840			return update;
1841	} while_for_each_ftrace_rec();
1842
1843	return update;
1844}
1845
1846static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1847				    int filter_hash)
1848{
1849	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1850}
1851
1852static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1853				   int filter_hash)
1854{
1855	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1856}
1857
1858static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
1859					  int filter_hash, int inc)
1860{
1861	struct ftrace_ops *op;
1862
1863	__ftrace_hash_rec_update(ops, filter_hash, inc);
1864
1865	if (ops->func_hash != &global_ops.local_hash)
1866		return;
1867
1868	/*
1869	 * If the ops shares the global_ops hash, then we need to update
1870	 * all ops that are enabled and use this hash.
1871	 */
1872	do_for_each_ftrace_op(op, ftrace_ops_list) {
1873		/* Already done */
1874		if (op == ops)
1875			continue;
1876		if (op->func_hash == &global_ops.local_hash)
1877			__ftrace_hash_rec_update(op, filter_hash, inc);
1878	} while_for_each_ftrace_op(op);
1879}
1880
1881static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
1882					   int filter_hash)
1883{
1884	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
1885}
1886
1887static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
1888					  int filter_hash)
1889{
1890	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
1891}
1892
1893/*
1894 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
1895 * or no-needed to update, -EBUSY if it detects a conflict of the flag
1896 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1897 * Note that old_hash and new_hash has below meanings
1898 *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1899 *  - If the hash is EMPTY_HASH, it hits nothing
1900 *  - Anything else hits the recs which match the hash entries.
1901 */
1902static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
1903					 struct ftrace_hash *old_hash,
1904					 struct ftrace_hash *new_hash)
1905{
1906	struct ftrace_page *pg;
1907	struct dyn_ftrace *rec, *end = NULL;
1908	int in_old, in_new;
1909
1910	/* Only update if the ops has been registered */
1911	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1912		return 0;
1913
1914	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
1915		return 0;
1916
1917	/*
1918	 * Since the IPMODIFY is a very address sensitive action, we do not
1919	 * allow ftrace_ops to set all functions to new hash.
1920	 */
1921	if (!new_hash || !old_hash)
1922		return -EINVAL;
1923
1924	/* Update rec->flags */
1925	do_for_each_ftrace_rec(pg, rec) {
1926
1927		if (rec->flags & FTRACE_FL_DISABLED)
1928			continue;
1929
1930		/* We need to update only differences of filter_hash */
1931		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1932		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1933		if (in_old == in_new)
1934			continue;
1935
1936		if (in_new) {
1937			/* New entries must ensure no others are using it */
1938			if (rec->flags & FTRACE_FL_IPMODIFY)
1939				goto rollback;
1940			rec->flags |= FTRACE_FL_IPMODIFY;
1941		} else /* Removed entry */
1942			rec->flags &= ~FTRACE_FL_IPMODIFY;
1943	} while_for_each_ftrace_rec();
1944
1945	return 0;
1946
1947rollback:
1948	end = rec;
1949
1950	/* Roll back what we did above */
1951	do_for_each_ftrace_rec(pg, rec) {
1952
1953		if (rec->flags & FTRACE_FL_DISABLED)
1954			continue;
1955
1956		if (rec == end)
1957			goto err_out;
1958
1959		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1960		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1961		if (in_old == in_new)
1962			continue;
1963
1964		if (in_new)
1965			rec->flags &= ~FTRACE_FL_IPMODIFY;
1966		else
1967			rec->flags |= FTRACE_FL_IPMODIFY;
1968	} while_for_each_ftrace_rec();
1969
1970err_out:
1971	return -EBUSY;
1972}
1973
1974static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
1975{
1976	struct ftrace_hash *hash = ops->func_hash->filter_hash;
1977
1978	if (ftrace_hash_empty(hash))
1979		hash = NULL;
1980
1981	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
1982}
1983
1984/* Disabling always succeeds */
1985static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
1986{
1987	struct ftrace_hash *hash = ops->func_hash->filter_hash;
1988
1989	if (ftrace_hash_empty(hash))
1990		hash = NULL;
1991
1992	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
1993}
1994
1995static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1996				       struct ftrace_hash *new_hash)
1997{
1998	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
1999
2000	if (ftrace_hash_empty(old_hash))
2001		old_hash = NULL;
2002
2003	if (ftrace_hash_empty(new_hash))
2004		new_hash = NULL;
2005
2006	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
2007}
2008
2009static void print_ip_ins(const char *fmt, const unsigned char *p)
2010{
2011	int i;
2012
2013	printk(KERN_CONT "%s", fmt);
2014
2015	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
2016		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
2017}
2018
2019static struct ftrace_ops *
2020ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
2021static struct ftrace_ops *
2022ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
2023
2024enum ftrace_bug_type ftrace_bug_type;
2025const void *ftrace_expected;
2026
2027static void print_bug_type(void)
2028{
2029	switch (ftrace_bug_type) {
2030	case FTRACE_BUG_UNKNOWN:
2031		break;
2032	case FTRACE_BUG_INIT:
2033		pr_info("Initializing ftrace call sites\n");
2034		break;
2035	case FTRACE_BUG_NOP:
2036		pr_info("Setting ftrace call site to NOP\n");
2037		break;
2038	case FTRACE_BUG_CALL:
2039		pr_info("Setting ftrace call site to call ftrace function\n");
2040		break;
2041	case FTRACE_BUG_UPDATE:
2042		pr_info("Updating ftrace call site to call a different ftrace function\n");
2043		break;
2044	}
2045}
2046
2047/**
2048 * ftrace_bug - report and shutdown function tracer
2049 * @failed: The failed type (EFAULT, EINVAL, EPERM)
2050 * @rec: The record that failed
2051 *
2052 * The arch code that enables or disables the function tracing
2053 * can call ftrace_bug() when it has detected a problem in
2054 * modifying the code. @failed should be one of either:
2055 * EFAULT - if the problem happens on reading the @ip address
2056 * EINVAL - if what is read at @ip is not what was expected
2057 * EPERM - if the problem happens on writting to the @ip address
2058 */
2059void ftrace_bug(int failed, struct dyn_ftrace *rec)
2060{
2061	unsigned long ip = rec ? rec->ip : 0;
2062
 
 
2063	switch (failed) {
2064	case -EFAULT:
2065		FTRACE_WARN_ON_ONCE(1);
2066		pr_info("ftrace faulted on modifying ");
2067		print_ip_sym(ip);
2068		break;
2069	case -EINVAL:
2070		FTRACE_WARN_ON_ONCE(1);
2071		pr_info("ftrace failed to modify ");
2072		print_ip_sym(ip);
2073		print_ip_ins(" actual:   ", (unsigned char *)ip);
2074		pr_cont("\n");
2075		if (ftrace_expected) {
2076			print_ip_ins(" expected: ", ftrace_expected);
2077			pr_cont("\n");
2078		}
2079		break;
2080	case -EPERM:
2081		FTRACE_WARN_ON_ONCE(1);
2082		pr_info("ftrace faulted on writing ");
2083		print_ip_sym(ip);
2084		break;
2085	default:
2086		FTRACE_WARN_ON_ONCE(1);
2087		pr_info("ftrace faulted on unknown error ");
2088		print_ip_sym(ip);
2089	}
2090	print_bug_type();
2091	if (rec) {
2092		struct ftrace_ops *ops = NULL;
2093
2094		pr_info("ftrace record flags: %lx\n", rec->flags);
2095		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
2096			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
2097		if (rec->flags & FTRACE_FL_TRAMP_EN) {
2098			ops = ftrace_find_tramp_ops_any(rec);
2099			if (ops) {
2100				do {
2101					pr_cont("\ttramp: %pS (%pS)",
2102						(void *)ops->trampoline,
2103						(void *)ops->func);
2104					ops = ftrace_find_tramp_ops_next(rec, ops);
2105				} while (ops);
2106			} else
2107				pr_cont("\ttramp: ERROR!");
2108
2109		}
2110		ip = ftrace_get_addr_curr(rec);
2111		pr_cont("\n expected tramp: %lx\n", ip);
2112	}
 
 
2113}
2114
2115static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
2116{
2117	unsigned long flag = 0UL;
2118
2119	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2120
2121	if (rec->flags & FTRACE_FL_DISABLED)
2122		return FTRACE_UPDATE_IGNORE;
2123
2124	/*
2125	 * If we are updating calls:
2126	 *
2127	 *   If the record has a ref count, then we need to enable it
2128	 *   because someone is using it.
2129	 *
2130	 *   Otherwise we make sure its disabled.
2131	 *
2132	 * If we are disabling calls, then disable all records that
2133	 * are enabled.
2134	 */
2135	if (enable && ftrace_rec_count(rec))
2136		flag = FTRACE_FL_ENABLED;
2137
2138	/*
2139	 * If enabling and the REGS flag does not match the REGS_EN, or
2140	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
2141	 * this record. Set flags to fail the compare against ENABLED.
 
2142	 */
2143	if (flag) {
2144		if (!(rec->flags & FTRACE_FL_REGS) != 
2145		    !(rec->flags & FTRACE_FL_REGS_EN))
2146			flag |= FTRACE_FL_REGS;
2147
2148		if (!(rec->flags & FTRACE_FL_TRAMP) != 
2149		    !(rec->flags & FTRACE_FL_TRAMP_EN))
2150			flag |= FTRACE_FL_TRAMP;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2151	}
2152
2153	/* If the state of this record hasn't changed, then do nothing */
2154	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2155		return FTRACE_UPDATE_IGNORE;
2156
2157	if (flag) {
2158		/* Save off if rec is being enabled (for return value) */
2159		flag ^= rec->flags & FTRACE_FL_ENABLED;
2160
2161		if (update) {
2162			rec->flags |= FTRACE_FL_ENABLED;
2163			if (flag & FTRACE_FL_REGS) {
2164				if (rec->flags & FTRACE_FL_REGS)
2165					rec->flags |= FTRACE_FL_REGS_EN;
2166				else
2167					rec->flags &= ~FTRACE_FL_REGS_EN;
2168			}
2169			if (flag & FTRACE_FL_TRAMP) {
2170				if (rec->flags & FTRACE_FL_TRAMP)
2171					rec->flags |= FTRACE_FL_TRAMP_EN;
2172				else
2173					rec->flags &= ~FTRACE_FL_TRAMP_EN;
2174			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2175		}
2176
2177		/*
2178		 * If this record is being updated from a nop, then
2179		 *   return UPDATE_MAKE_CALL.
2180		 * Otherwise,
2181		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2182		 *   from the save regs, to a non-save regs function or
2183		 *   vice versa, or from a trampoline call.
2184		 */
2185		if (flag & FTRACE_FL_ENABLED) {
2186			ftrace_bug_type = FTRACE_BUG_CALL;
2187			return FTRACE_UPDATE_MAKE_CALL;
2188		}
2189
2190		ftrace_bug_type = FTRACE_BUG_UPDATE;
2191		return FTRACE_UPDATE_MODIFY_CALL;
2192	}
2193
2194	if (update) {
2195		/* If there's no more users, clear all flags */
2196		if (!ftrace_rec_count(rec))
2197			rec->flags = 0;
2198		else
2199			/*
2200			 * Just disable the record, but keep the ops TRAMP
2201			 * and REGS states. The _EN flags must be disabled though.
2202			 */
2203			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
2204					FTRACE_FL_REGS_EN);
2205	}
2206
2207	ftrace_bug_type = FTRACE_BUG_NOP;
2208	return FTRACE_UPDATE_MAKE_NOP;
2209}
2210
2211/**
2212 * ftrace_update_record, set a record that now is tracing or not
2213 * @rec: the record to update
2214 * @enable: set to 1 if the record is tracing, zero to force disable
2215 *
2216 * The records that represent all functions that can be traced need
2217 * to be updated when tracing has been enabled.
2218 */
2219int ftrace_update_record(struct dyn_ftrace *rec, int enable)
2220{
2221	return ftrace_check_record(rec, enable, 1);
2222}
2223
2224/**
2225 * ftrace_test_record, check if the record has been enabled or not
2226 * @rec: the record to test
2227 * @enable: set to 1 to check if enabled, 0 if it is disabled
2228 *
2229 * The arch code may need to test if a record is already set to
2230 * tracing to determine how to modify the function code that it
2231 * represents.
2232 */
2233int ftrace_test_record(struct dyn_ftrace *rec, int enable)
2234{
2235	return ftrace_check_record(rec, enable, 0);
2236}
2237
2238static struct ftrace_ops *
2239ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
2240{
2241	struct ftrace_ops *op;
2242	unsigned long ip = rec->ip;
2243
2244	do_for_each_ftrace_op(op, ftrace_ops_list) {
2245
2246		if (!op->trampoline)
2247			continue;
2248
2249		if (hash_contains_ip(ip, op->func_hash))
2250			return op;
2251	} while_for_each_ftrace_op(op);
2252
2253	return NULL;
2254}
2255
2256static struct ftrace_ops *
2257ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
2258			   struct ftrace_ops *op)
2259{
2260	unsigned long ip = rec->ip;
2261
2262	while_for_each_ftrace_op(op) {
2263
2264		if (!op->trampoline)
2265			continue;
2266
2267		if (hash_contains_ip(ip, op->func_hash))
2268			return op;
2269	} 
2270
2271	return NULL;
2272}
2273
2274static struct ftrace_ops *
2275ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
2276{
2277	struct ftrace_ops *op;
2278	unsigned long ip = rec->ip;
2279
2280	/*
2281	 * Need to check removed ops first.
2282	 * If they are being removed, and this rec has a tramp,
2283	 * and this rec is in the ops list, then it would be the
2284	 * one with the tramp.
2285	 */
2286	if (removed_ops) {
2287		if (hash_contains_ip(ip, &removed_ops->old_hash))
2288			return removed_ops;
2289	}
2290
2291	/*
2292	 * Need to find the current trampoline for a rec.
2293	 * Now, a trampoline is only attached to a rec if there
2294	 * was a single 'ops' attached to it. But this can be called
2295	 * when we are adding another op to the rec or removing the
2296	 * current one. Thus, if the op is being added, we can
2297	 * ignore it because it hasn't attached itself to the rec
2298	 * yet.
2299	 *
2300	 * If an ops is being modified (hooking to different functions)
2301	 * then we don't care about the new functions that are being
2302	 * added, just the old ones (that are probably being removed).
2303	 *
2304	 * If we are adding an ops to a function that already is using
2305	 * a trampoline, it needs to be removed (trampolines are only
2306	 * for single ops connected), then an ops that is not being
2307	 * modified also needs to be checked.
2308	 */
2309	do_for_each_ftrace_op(op, ftrace_ops_list) {
2310
2311		if (!op->trampoline)
2312			continue;
2313
2314		/*
2315		 * If the ops is being added, it hasn't gotten to
2316		 * the point to be removed from this tree yet.
2317		 */
2318		if (op->flags & FTRACE_OPS_FL_ADDING)
2319			continue;
2320
2321
2322		/*
2323		 * If the ops is being modified and is in the old
2324		 * hash, then it is probably being removed from this
2325		 * function.
2326		 */
2327		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
2328		    hash_contains_ip(ip, &op->old_hash))
2329			return op;
2330		/*
2331		 * If the ops is not being added or modified, and it's
2332		 * in its normal filter hash, then this must be the one
2333		 * we want!
2334		 */
2335		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
2336		    hash_contains_ip(ip, op->func_hash))
2337			return op;
2338
2339	} while_for_each_ftrace_op(op);
2340
2341	return NULL;
2342}
2343
2344static struct ftrace_ops *
2345ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
2346{
2347	struct ftrace_ops *op;
2348	unsigned long ip = rec->ip;
2349
2350	do_for_each_ftrace_op(op, ftrace_ops_list) {
2351		/* pass rec in as regs to have non-NULL val */
2352		if (hash_contains_ip(ip, op->func_hash))
2353			return op;
2354	} while_for_each_ftrace_op(op);
2355
2356	return NULL;
2357}
2358
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2359/**
2360 * ftrace_get_addr_new - Get the call address to set to
2361 * @rec:  The ftrace record descriptor
2362 *
2363 * If the record has the FTRACE_FL_REGS set, that means that it
2364 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
2365 * is not not set, then it wants to convert to the normal callback.
2366 *
2367 * Returns the address of the trampoline to set to
2368 */
2369unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
2370{
2371	struct ftrace_ops *ops;
 
 
 
 
 
 
 
 
 
2372
2373	/* Trampolines take precedence over regs */
2374	if (rec->flags & FTRACE_FL_TRAMP) {
2375		ops = ftrace_find_tramp_ops_new(rec);
2376		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2377			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
2378				(void *)rec->ip, (void *)rec->ip, rec->flags);
2379			/* Ftrace is shutting down, return anything */
2380			return (unsigned long)FTRACE_ADDR;
2381		}
2382		return ops->trampoline;
2383	}
2384
2385	if (rec->flags & FTRACE_FL_REGS)
2386		return (unsigned long)FTRACE_REGS_ADDR;
2387	else
2388		return (unsigned long)FTRACE_ADDR;
2389}
2390
2391/**
2392 * ftrace_get_addr_curr - Get the call address that is already there
2393 * @rec:  The ftrace record descriptor
2394 *
2395 * The FTRACE_FL_REGS_EN is set when the record already points to
2396 * a function that saves all the regs. Basically the '_EN' version
2397 * represents the current state of the function.
2398 *
2399 * Returns the address of the trampoline that is currently being called
2400 */
2401unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
2402{
2403	struct ftrace_ops *ops;
 
 
 
 
 
 
 
 
 
2404
2405	/* Trampolines take precedence over regs */
2406	if (rec->flags & FTRACE_FL_TRAMP_EN) {
2407		ops = ftrace_find_tramp_ops_curr(rec);
2408		if (FTRACE_WARN_ON(!ops)) {
2409			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
2410				(void *)rec->ip, (void *)rec->ip);
2411			/* Ftrace is shutting down, return anything */
2412			return (unsigned long)FTRACE_ADDR;
2413		}
2414		return ops->trampoline;
2415	}
2416
2417	if (rec->flags & FTRACE_FL_REGS_EN)
2418		return (unsigned long)FTRACE_REGS_ADDR;
2419	else
2420		return (unsigned long)FTRACE_ADDR;
2421}
2422
2423static int
2424__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
2425{
2426	unsigned long ftrace_old_addr;
2427	unsigned long ftrace_addr;
2428	int ret;
2429
2430	ftrace_addr = ftrace_get_addr_new(rec);
2431
2432	/* This needs to be done before we call ftrace_update_record */
2433	ftrace_old_addr = ftrace_get_addr_curr(rec);
2434
2435	ret = ftrace_update_record(rec, enable);
2436
2437	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2438
2439	switch (ret) {
2440	case FTRACE_UPDATE_IGNORE:
2441		return 0;
2442
2443	case FTRACE_UPDATE_MAKE_CALL:
2444		ftrace_bug_type = FTRACE_BUG_CALL;
2445		return ftrace_make_call(rec, ftrace_addr);
2446
2447	case FTRACE_UPDATE_MAKE_NOP:
2448		ftrace_bug_type = FTRACE_BUG_NOP;
2449		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2450
2451	case FTRACE_UPDATE_MODIFY_CALL:
2452		ftrace_bug_type = FTRACE_BUG_UPDATE;
2453		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2454	}
2455
2456	return -1; /* unknow ftrace bug */
2457}
2458
2459void __weak ftrace_replace_code(int enable)
2460{
2461	struct dyn_ftrace *rec;
2462	struct ftrace_page *pg;
 
 
2463	int failed;
2464
2465	if (unlikely(ftrace_disabled))
2466		return;
2467
2468	do_for_each_ftrace_rec(pg, rec) {
2469
2470		if (rec->flags & FTRACE_FL_DISABLED)
2471			continue;
2472
2473		failed = __ftrace_replace_code(rec, enable);
2474		if (failed) {
2475			ftrace_bug(failed, rec);
2476			/* Stop processing */
2477			return;
2478		}
 
 
2479	} while_for_each_ftrace_rec();
2480}
2481
2482struct ftrace_rec_iter {
2483	struct ftrace_page	*pg;
2484	int			index;
2485};
2486
2487/**
2488 * ftrace_rec_iter_start, start up iterating over traced functions
2489 *
2490 * Returns an iterator handle that is used to iterate over all
2491 * the records that represent address locations where functions
2492 * are traced.
2493 *
2494 * May return NULL if no records are available.
2495 */
2496struct ftrace_rec_iter *ftrace_rec_iter_start(void)
2497{
2498	/*
2499	 * We only use a single iterator.
2500	 * Protected by the ftrace_lock mutex.
2501	 */
2502	static struct ftrace_rec_iter ftrace_rec_iter;
2503	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
2504
2505	iter->pg = ftrace_pages_start;
2506	iter->index = 0;
2507
2508	/* Could have empty pages */
2509	while (iter->pg && !iter->pg->index)
2510		iter->pg = iter->pg->next;
2511
2512	if (!iter->pg)
2513		return NULL;
2514
2515	return iter;
2516}
2517
2518/**
2519 * ftrace_rec_iter_next, get the next record to process.
2520 * @iter: The handle to the iterator.
2521 *
2522 * Returns the next iterator after the given iterator @iter.
2523 */
2524struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
2525{
2526	iter->index++;
2527
2528	if (iter->index >= iter->pg->index) {
2529		iter->pg = iter->pg->next;
2530		iter->index = 0;
2531
2532		/* Could have empty pages */
2533		while (iter->pg && !iter->pg->index)
2534			iter->pg = iter->pg->next;
2535	}
2536
2537	if (!iter->pg)
2538		return NULL;
2539
2540	return iter;
2541}
2542
2543/**
2544 * ftrace_rec_iter_record, get the record at the iterator location
2545 * @iter: The current iterator location
2546 *
2547 * Returns the record that the current @iter is at.
2548 */
2549struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
2550{
2551	return &iter->pg->records[iter->index];
2552}
2553
2554static int
2555ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
2556{
2557	int ret;
2558
2559	if (unlikely(ftrace_disabled))
2560		return 0;
2561
2562	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
2563	if (ret) {
2564		ftrace_bug_type = FTRACE_BUG_INIT;
2565		ftrace_bug(ret, rec);
2566		return 0;
2567	}
2568	return 1;
2569}
2570
2571/*
2572 * archs can override this function if they must do something
2573 * before the modifying code is performed.
2574 */
2575int __weak ftrace_arch_code_modify_prepare(void)
2576{
2577	return 0;
2578}
2579
2580/*
2581 * archs can override this function if they must do something
2582 * after the modifying code is performed.
2583 */
2584int __weak ftrace_arch_code_modify_post_process(void)
2585{
2586	return 0;
2587}
2588
2589void ftrace_modify_all_code(int command)
2590{
2591	int update = command & FTRACE_UPDATE_TRACE_FUNC;
 
2592	int err = 0;
2593
 
 
 
2594	/*
2595	 * If the ftrace_caller calls a ftrace_ops func directly,
2596	 * we need to make sure that it only traces functions it
2597	 * expects to trace. When doing the switch of functions,
2598	 * we need to update to the ftrace_ops_list_func first
2599	 * before the transition between old and new calls are set,
2600	 * as the ftrace_ops_list_func will check the ops hashes
2601	 * to make sure the ops are having the right functions
2602	 * traced.
2603	 */
2604	if (update) {
2605		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
2606		if (FTRACE_WARN_ON(err))
2607			return;
2608	}
2609
2610	if (command & FTRACE_UPDATE_CALLS)
2611		ftrace_replace_code(1);
2612	else if (command & FTRACE_DISABLE_CALLS)
2613		ftrace_replace_code(0);
2614
2615	if (update && ftrace_trace_function != ftrace_ops_list_func) {
2616		function_trace_op = set_function_trace_op;
2617		smp_wmb();
2618		/* If irqs are disabled, we are in stop machine */
2619		if (!irqs_disabled())
2620			smp_call_function(ftrace_sync_ipi, NULL, 1);
2621		err = ftrace_update_ftrace_func(ftrace_trace_function);
2622		if (FTRACE_WARN_ON(err))
2623			return;
2624	}
2625
2626	if (command & FTRACE_START_FUNC_RET)
2627		err = ftrace_enable_ftrace_graph_caller();
2628	else if (command & FTRACE_STOP_FUNC_RET)
2629		err = ftrace_disable_ftrace_graph_caller();
2630	FTRACE_WARN_ON(err);
2631}
2632
2633static int __ftrace_modify_code(void *data)
2634{
2635	int *command = data;
2636
2637	ftrace_modify_all_code(*command);
2638
2639	return 0;
2640}
2641
2642/**
2643 * ftrace_run_stop_machine, go back to the stop machine method
2644 * @command: The command to tell ftrace what to do
2645 *
2646 * If an arch needs to fall back to the stop machine method, the
2647 * it can call this function.
2648 */
2649void ftrace_run_stop_machine(int command)
2650{
2651	stop_machine(__ftrace_modify_code, &command, NULL);
2652}
2653
2654/**
2655 * arch_ftrace_update_code, modify the code to trace or not trace
2656 * @command: The command that needs to be done
2657 *
2658 * Archs can override this function if it does not need to
2659 * run stop_machine() to modify code.
2660 */
2661void __weak arch_ftrace_update_code(int command)
2662{
2663	ftrace_run_stop_machine(command);
2664}
2665
2666static void ftrace_run_update_code(int command)
2667{
2668	int ret;
2669
2670	ret = ftrace_arch_code_modify_prepare();
2671	FTRACE_WARN_ON(ret);
2672	if (ret)
2673		return;
2674
2675	/*
2676	 * By default we use stop_machine() to modify the code.
2677	 * But archs can do what ever they want as long as it
2678	 * is safe. The stop_machine() is the safest, but also
2679	 * produces the most overhead.
2680	 */
2681	arch_ftrace_update_code(command);
2682
2683	ret = ftrace_arch_code_modify_post_process();
2684	FTRACE_WARN_ON(ret);
2685}
2686
2687static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2688				   struct ftrace_ops_hash *old_hash)
2689{
2690	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2691	ops->old_hash.filter_hash = old_hash->filter_hash;
2692	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2693	ftrace_run_update_code(command);
2694	ops->old_hash.filter_hash = NULL;
2695	ops->old_hash.notrace_hash = NULL;
2696	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
2697}
2698
2699static ftrace_func_t saved_ftrace_func;
2700static int ftrace_start_up;
2701
2702void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
2703{
2704}
2705
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2706static void ftrace_startup_enable(int command)
2707{
2708	if (saved_ftrace_func != ftrace_trace_function) {
2709		saved_ftrace_func = ftrace_trace_function;
2710		command |= FTRACE_UPDATE_TRACE_FUNC;
2711	}
2712
2713	if (!command || !ftrace_enabled)
2714		return;
2715
2716	ftrace_run_update_code(command);
2717}
2718
2719static void ftrace_startup_all(int command)
2720{
2721	update_all_ops = true;
2722	ftrace_startup_enable(command);
2723	update_all_ops = false;
2724}
2725
2726static int ftrace_startup(struct ftrace_ops *ops, int command)
2727{
2728	int ret;
2729
2730	if (unlikely(ftrace_disabled))
2731		return -ENODEV;
2732
2733	ret = __register_ftrace_function(ops);
2734	if (ret)
2735		return ret;
2736
2737	ftrace_start_up++;
2738
2739	/*
2740	 * Note that ftrace probes uses this to start up
2741	 * and modify functions it will probe. But we still
2742	 * set the ADDING flag for modification, as probes
2743	 * do not have trampolines. If they add them in the
2744	 * future, then the probes will need to distinguish
2745	 * between adding and updating probes.
2746	 */
2747	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2748
2749	ret = ftrace_hash_ipmodify_enable(ops);
2750	if (ret < 0) {
2751		/* Rollback registration process */
2752		__unregister_ftrace_function(ops);
2753		ftrace_start_up--;
2754		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 
 
2755		return ret;
2756	}
2757
2758	if (ftrace_hash_rec_enable(ops, 1))
2759		command |= FTRACE_UPDATE_CALLS;
2760
2761	ftrace_startup_enable(command);
2762
2763	ops->flags &= ~FTRACE_OPS_FL_ADDING;
2764
2765	return 0;
2766}
2767
2768static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2769{
2770	int ret;
2771
2772	if (unlikely(ftrace_disabled))
2773		return -ENODEV;
2774
2775	ret = __unregister_ftrace_function(ops);
2776	if (ret)
2777		return ret;
2778
2779	ftrace_start_up--;
2780	/*
2781	 * Just warn in case of unbalance, no need to kill ftrace, it's not
2782	 * critical but the ftrace_call callers may be never nopped again after
2783	 * further ftrace uses.
2784	 */
2785	WARN_ON_ONCE(ftrace_start_up < 0);
2786
2787	/* Disabling ipmodify never fails */
2788	ftrace_hash_ipmodify_disable(ops);
2789
2790	if (ftrace_hash_rec_disable(ops, 1))
2791		command |= FTRACE_UPDATE_CALLS;
2792
2793	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2794
2795	if (saved_ftrace_func != ftrace_trace_function) {
2796		saved_ftrace_func = ftrace_trace_function;
2797		command |= FTRACE_UPDATE_TRACE_FUNC;
2798	}
2799
2800	if (!command || !ftrace_enabled) {
2801		/*
2802		 * If these are dynamic or per_cpu ops, they still
2803		 * need their data freed. Since, function tracing is
2804		 * not currently active, we can just free them
2805		 * without synchronizing all CPUs.
2806		 */
2807		if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
2808			goto free_ops;
2809
2810		return 0;
2811	}
2812
2813	/*
2814	 * If the ops uses a trampoline, then it needs to be
2815	 * tested first on update.
2816	 */
2817	ops->flags |= FTRACE_OPS_FL_REMOVING;
2818	removed_ops = ops;
2819
2820	/* The trampoline logic checks the old hashes */
2821	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
2822	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
2823
2824	ftrace_run_update_code(command);
2825
2826	/*
2827	 * If there's no more ops registered with ftrace, run a
2828	 * sanity check to make sure all rec flags are cleared.
2829	 */
2830	if (rcu_dereference_protected(ftrace_ops_list,
2831			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
2832		struct ftrace_page *pg;
2833		struct dyn_ftrace *rec;
2834
2835		do_for_each_ftrace_rec(pg, rec) {
2836			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
2837				pr_warn("  %pS flags:%lx\n",
2838					(void *)rec->ip, rec->flags);
2839		} while_for_each_ftrace_rec();
2840	}
2841
2842	ops->old_hash.filter_hash = NULL;
2843	ops->old_hash.notrace_hash = NULL;
2844
2845	removed_ops = NULL;
2846	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2847
2848	/*
2849	 * Dynamic ops may be freed, we must make sure that all
2850	 * callers are done before leaving this function.
2851	 * The same goes for freeing the per_cpu data of the per_cpu
2852	 * ops.
2853	 */
2854	if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
2855		/*
2856		 * We need to do a hard force of sched synchronization.
2857		 * This is because we use preempt_disable() to do RCU, but
2858		 * the function tracers can be called where RCU is not watching
2859		 * (like before user_exit()). We can not rely on the RCU
2860		 * infrastructure to do the synchronization, thus we must do it
2861		 * ourselves.
2862		 */
2863		schedule_on_each_cpu(ftrace_sync);
2864
2865		/*
2866		 * When the kernel is preeptive, tasks can be preempted
2867		 * while on a ftrace trampoline. Just scheduling a task on
2868		 * a CPU is not good enough to flush them. Calling
2869		 * synchornize_rcu_tasks() will wait for those tasks to
2870		 * execute and either schedule voluntarily or enter user space.
2871		 */
2872		if (IS_ENABLED(CONFIG_PREEMPT))
2873			synchronize_rcu_tasks();
2874
2875 free_ops:
2876		arch_ftrace_trampoline_free(ops);
2877	}
2878
2879	return 0;
2880}
2881
2882static void ftrace_startup_sysctl(void)
2883{
2884	int command;
2885
2886	if (unlikely(ftrace_disabled))
2887		return;
2888
2889	/* Force update next time */
2890	saved_ftrace_func = NULL;
2891	/* ftrace_start_up is true if we want ftrace running */
2892	if (ftrace_start_up) {
2893		command = FTRACE_UPDATE_CALLS;
2894		if (ftrace_graph_active)
2895			command |= FTRACE_START_FUNC_RET;
2896		ftrace_startup_enable(command);
2897	}
2898}
2899
2900static void ftrace_shutdown_sysctl(void)
2901{
2902	int command;
2903
2904	if (unlikely(ftrace_disabled))
2905		return;
2906
2907	/* ftrace_start_up is true if ftrace is running */
2908	if (ftrace_start_up) {
2909		command = FTRACE_DISABLE_CALLS;
2910		if (ftrace_graph_active)
2911			command |= FTRACE_STOP_FUNC_RET;
2912		ftrace_run_update_code(command);
2913	}
2914}
2915
2916static u64		ftrace_update_time;
2917unsigned long		ftrace_update_tot_cnt;
 
 
2918
2919static inline int ops_traces_mod(struct ftrace_ops *ops)
2920{
2921	/*
2922	 * Filter_hash being empty will default to trace module.
2923	 * But notrace hash requires a test of individual module functions.
2924	 */
2925	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
2926		ftrace_hash_empty(ops->func_hash->notrace_hash);
2927}
2928
2929/*
2930 * Check if the current ops references the record.
2931 *
2932 * If the ops traces all functions, then it was already accounted for.
2933 * If the ops does not trace the current record function, skip it.
2934 * If the ops ignores the function via notrace filter, skip it.
2935 */
2936static inline bool
2937ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
2938{
2939	/* If ops isn't enabled, ignore it */
2940	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
2941		return 0;
2942
2943	/* If ops traces all then it includes this function */
2944	if (ops_traces_mod(ops))
2945		return 1;
2946
2947	/* The function must be in the filter */
2948	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2949	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2950		return 0;
2951
2952	/* If in notrace hash, we ignore it too */
2953	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2954		return 0;
2955
2956	return 1;
2957}
2958
2959static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2960{
2961	struct ftrace_page *pg;
2962	struct dyn_ftrace *p;
2963	u64 start, stop;
2964	unsigned long update_cnt = 0;
2965	unsigned long rec_flags = 0;
2966	int i;
2967
2968	start = ftrace_now(raw_smp_processor_id());
2969
2970	/*
2971	 * When a module is loaded, this function is called to convert
2972	 * the calls to mcount in its text to nops, and also to create
2973	 * an entry in the ftrace data. Now, if ftrace is activated
2974	 * after this call, but before the module sets its text to
2975	 * read-only, the modification of enabling ftrace can fail if
2976	 * the read-only is done while ftrace is converting the calls.
2977	 * To prevent this, the module's records are set as disabled
2978	 * and will be enabled after the call to set the module's text
2979	 * to read-only.
2980	 */
2981	if (mod)
2982		rec_flags |= FTRACE_FL_DISABLED;
2983
2984	for (pg = new_pgs; pg; pg = pg->next) {
2985
2986		for (i = 0; i < pg->index; i++) {
2987
2988			/* If something went wrong, bail without enabling anything */
2989			if (unlikely(ftrace_disabled))
2990				return -1;
2991
2992			p = &pg->records[i];
2993			p->flags = rec_flags;
2994
2995			/*
2996			 * Do the initial record conversion from mcount jump
2997			 * to the NOP instructions.
2998			 */
2999			if (!ftrace_code_disable(mod, p))
 
3000				break;
3001
3002			update_cnt++;
3003		}
3004	}
3005
3006	stop = ftrace_now(raw_smp_processor_id());
3007	ftrace_update_time = stop - start;
3008	ftrace_update_tot_cnt += update_cnt;
3009
3010	return 0;
3011}
3012
3013static int ftrace_allocate_records(struct ftrace_page *pg, int count)
3014{
3015	int order;
3016	int cnt;
3017
3018	if (WARN_ON(!count))
3019		return -EINVAL;
3020
3021	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
3022
3023	/*
3024	 * We want to fill as much as possible. No more than a page
3025	 * may be empty.
3026	 */
3027	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
3028		order--;
3029
3030 again:
3031	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
3032
3033	if (!pg->records) {
3034		/* if we can't allocate this size, try something smaller */
3035		if (!order)
3036			return -ENOMEM;
3037		order >>= 1;
3038		goto again;
3039	}
3040
 
 
 
3041	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
3042	pg->size = cnt;
3043
3044	if (cnt > count)
3045		cnt = count;
3046
3047	return cnt;
3048}
3049
3050static struct ftrace_page *
3051ftrace_allocate_pages(unsigned long num_to_init)
3052{
3053	struct ftrace_page *start_pg;
3054	struct ftrace_page *pg;
3055	int order;
3056	int cnt;
3057
3058	if (!num_to_init)
3059		return 0;
3060
3061	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
3062	if (!pg)
3063		return NULL;
3064
3065	/*
3066	 * Try to allocate as much as possible in one continues
3067	 * location that fills in all of the space. We want to
3068	 * waste as little space as possible.
3069	 */
3070	for (;;) {
3071		cnt = ftrace_allocate_records(pg, num_to_init);
3072		if (cnt < 0)
3073			goto free_pages;
3074
3075		num_to_init -= cnt;
3076		if (!num_to_init)
3077			break;
3078
3079		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
3080		if (!pg->next)
3081			goto free_pages;
3082
3083		pg = pg->next;
3084	}
3085
3086	return start_pg;
3087
3088 free_pages:
3089	pg = start_pg;
3090	while (pg) {
3091		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
3092		free_pages((unsigned long)pg->records, order);
3093		start_pg = pg->next;
3094		kfree(pg);
3095		pg = start_pg;
 
 
3096	}
3097	pr_info("ftrace: FAILED to allocate memory for functions\n");
3098	return NULL;
3099}
3100
3101#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
3102
3103struct ftrace_iterator {
3104	loff_t				pos;
3105	loff_t				func_pos;
3106	loff_t				mod_pos;
3107	struct ftrace_page		*pg;
3108	struct dyn_ftrace		*func;
3109	struct ftrace_func_probe	*probe;
3110	struct ftrace_func_entry	*probe_entry;
3111	struct trace_parser		parser;
3112	struct ftrace_hash		*hash;
3113	struct ftrace_ops		*ops;
3114	struct trace_array		*tr;
3115	struct list_head		*mod_list;
3116	int				pidx;
3117	int				idx;
3118	unsigned			flags;
3119};
3120
3121static void *
3122t_probe_next(struct seq_file *m, loff_t *pos)
3123{
3124	struct ftrace_iterator *iter = m->private;
3125	struct trace_array *tr = iter->ops->private;
3126	struct list_head *func_probes;
3127	struct ftrace_hash *hash;
3128	struct list_head *next;
3129	struct hlist_node *hnd = NULL;
3130	struct hlist_head *hhd;
3131	int size;
3132
3133	(*pos)++;
3134	iter->pos = *pos;
3135
3136	if (!tr)
3137		return NULL;
3138
3139	func_probes = &tr->func_probes;
3140	if (list_empty(func_probes))
3141		return NULL;
3142
3143	if (!iter->probe) {
3144		next = func_probes->next;
3145		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3146	}
3147
3148	if (iter->probe_entry)
3149		hnd = &iter->probe_entry->hlist;
3150
3151	hash = iter->probe->ops.func_hash->filter_hash;
 
 
 
 
 
 
 
 
3152	size = 1 << hash->size_bits;
3153
3154 retry:
3155	if (iter->pidx >= size) {
3156		if (iter->probe->list.next == func_probes)
3157			return NULL;
3158		next = iter->probe->list.next;
3159		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3160		hash = iter->probe->ops.func_hash->filter_hash;
3161		size = 1 << hash->size_bits;
3162		iter->pidx = 0;
3163	}
3164
3165	hhd = &hash->buckets[iter->pidx];
3166
3167	if (hlist_empty(hhd)) {
3168		iter->pidx++;
3169		hnd = NULL;
3170		goto retry;
3171	}
3172
3173	if (!hnd)
3174		hnd = hhd->first;
3175	else {
3176		hnd = hnd->next;
3177		if (!hnd) {
3178			iter->pidx++;
3179			goto retry;
3180		}
3181	}
3182
3183	if (WARN_ON_ONCE(!hnd))
3184		return NULL;
3185
3186	iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
3187
3188	return iter;
3189}
3190
3191static void *t_probe_start(struct seq_file *m, loff_t *pos)
3192{
3193	struct ftrace_iterator *iter = m->private;
3194	void *p = NULL;
3195	loff_t l;
3196
3197	if (!(iter->flags & FTRACE_ITER_DO_PROBES))
3198		return NULL;
3199
3200	if (iter->mod_pos > *pos)
3201		return NULL;
3202
3203	iter->probe = NULL;
3204	iter->probe_entry = NULL;
3205	iter->pidx = 0;
3206	for (l = 0; l <= (*pos - iter->mod_pos); ) {
3207		p = t_probe_next(m, &l);
3208		if (!p)
3209			break;
3210	}
3211	if (!p)
3212		return NULL;
3213
3214	/* Only set this if we have an item */
3215	iter->flags |= FTRACE_ITER_PROBE;
3216
3217	return iter;
3218}
3219
3220static int
3221t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
3222{
3223	struct ftrace_func_entry *probe_entry;
3224	struct ftrace_probe_ops *probe_ops;
3225	struct ftrace_func_probe *probe;
3226
3227	probe = iter->probe;
3228	probe_entry = iter->probe_entry;
3229
3230	if (WARN_ON_ONCE(!probe || !probe_entry))
3231		return -EIO;
3232
3233	probe_ops = probe->probe_ops;
3234
3235	if (probe_ops->print)
3236		return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
3237
3238	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
3239		   (void *)probe_ops->func);
3240
3241	return 0;
3242}
3243
3244static void *
3245t_mod_next(struct seq_file *m, loff_t *pos)
3246{
3247	struct ftrace_iterator *iter = m->private;
3248	struct trace_array *tr = iter->tr;
3249
3250	(*pos)++;
3251	iter->pos = *pos;
3252
3253	iter->mod_list = iter->mod_list->next;
3254
3255	if (iter->mod_list == &tr->mod_trace ||
3256	    iter->mod_list == &tr->mod_notrace) {
3257		iter->flags &= ~FTRACE_ITER_MOD;
3258		return NULL;
3259	}
3260
3261	iter->mod_pos = *pos;
3262
3263	return iter;
3264}
3265
3266static void *t_mod_start(struct seq_file *m, loff_t *pos)
3267{
3268	struct ftrace_iterator *iter = m->private;
3269	void *p = NULL;
3270	loff_t l;
3271
3272	if (iter->func_pos > *pos)
3273		return NULL;
3274
3275	iter->mod_pos = iter->func_pos;
3276
3277	/* probes are only available if tr is set */
3278	if (!iter->tr)
3279		return NULL;
3280
3281	for (l = 0; l <= (*pos - iter->func_pos); ) {
3282		p = t_mod_next(m, &l);
3283		if (!p)
3284			break;
3285	}
3286	if (!p) {
3287		iter->flags &= ~FTRACE_ITER_MOD;
3288		return t_probe_start(m, pos);
3289	}
3290
3291	/* Only set this if we have an item */
3292	iter->flags |= FTRACE_ITER_MOD;
3293
3294	return iter;
3295}
3296
3297static int
3298t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
3299{
3300	struct ftrace_mod_load *ftrace_mod;
3301	struct trace_array *tr = iter->tr;
3302
3303	if (WARN_ON_ONCE(!iter->mod_list) ||
3304			 iter->mod_list == &tr->mod_trace ||
3305			 iter->mod_list == &tr->mod_notrace)
3306		return -EIO;
3307
3308	ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);
3309
3310	if (ftrace_mod->func)
3311		seq_printf(m, "%s", ftrace_mod->func);
3312	else
3313		seq_putc(m, '*');
3314
3315	seq_printf(m, ":mod:%s\n", ftrace_mod->module);
3316
3317	return 0;
3318}
3319
3320static void *
3321t_func_next(struct seq_file *m, loff_t *pos)
3322{
3323	struct ftrace_iterator *iter = m->private;
3324	struct dyn_ftrace *rec = NULL;
3325
3326	(*pos)++;
3327
3328 retry:
3329	if (iter->idx >= iter->pg->index) {
3330		if (iter->pg->next) {
3331			iter->pg = iter->pg->next;
3332			iter->idx = 0;
3333			goto retry;
3334		}
3335	} else {
3336		rec = &iter->pg->records[iter->idx++];
3337		if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
3338		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||
3339
3340		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3341		     !(rec->flags & FTRACE_FL_ENABLED))) {
3342
3343			rec = NULL;
3344			goto retry;
3345		}
3346	}
3347
3348	if (!rec)
3349		return NULL;
3350
3351	iter->pos = iter->func_pos = *pos;
3352	iter->func = rec;
3353
3354	return iter;
3355}
3356
3357static void *
3358t_next(struct seq_file *m, void *v, loff_t *pos)
3359{
3360	struct ftrace_iterator *iter = m->private;
3361	loff_t l = *pos; /* t_probe_start() must use original pos */
3362	void *ret;
3363
3364	if (unlikely(ftrace_disabled))
3365		return NULL;
3366
3367	if (iter->flags & FTRACE_ITER_PROBE)
3368		return t_probe_next(m, pos);
3369
3370	if (iter->flags & FTRACE_ITER_MOD)
3371		return t_mod_next(m, pos);
3372
3373	if (iter->flags & FTRACE_ITER_PRINTALL) {
3374		/* next must increment pos, and t_probe_start does not */
3375		(*pos)++;
3376		return t_mod_start(m, &l);
3377	}
3378
3379	ret = t_func_next(m, pos);
3380
3381	if (!ret)
3382		return t_mod_start(m, &l);
3383
3384	return ret;
3385}
3386
3387static void reset_iter_read(struct ftrace_iterator *iter)
3388{
3389	iter->pos = 0;
3390	iter->func_pos = 0;
3391	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
3392}
3393
3394static void *t_start(struct seq_file *m, loff_t *pos)
3395{
3396	struct ftrace_iterator *iter = m->private;
3397	void *p = NULL;
3398	loff_t l;
3399
3400	mutex_lock(&ftrace_lock);
3401
3402	if (unlikely(ftrace_disabled))
3403		return NULL;
3404
3405	/*
3406	 * If an lseek was done, then reset and start from beginning.
3407	 */
3408	if (*pos < iter->pos)
3409		reset_iter_read(iter);
3410
3411	/*
3412	 * For set_ftrace_filter reading, if we have the filter
3413	 * off, we can short cut and just print out that all
3414	 * functions are enabled.
3415	 */
3416	if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
3417	    ftrace_hash_empty(iter->hash)) {
3418		iter->func_pos = 1; /* Account for the message */
3419		if (*pos > 0)
3420			return t_mod_start(m, pos);
3421		iter->flags |= FTRACE_ITER_PRINTALL;
3422		/* reset in case of seek/pread */
3423		iter->flags &= ~FTRACE_ITER_PROBE;
3424		return iter;
3425	}
3426
3427	if (iter->flags & FTRACE_ITER_MOD)
3428		return t_mod_start(m, pos);
3429
3430	/*
3431	 * Unfortunately, we need to restart at ftrace_pages_start
3432	 * every time we let go of the ftrace_mutex. This is because
3433	 * those pointers can change without the lock.
3434	 */
3435	iter->pg = ftrace_pages_start;
3436	iter->idx = 0;
3437	for (l = 0; l <= *pos; ) {
3438		p = t_func_next(m, &l);
3439		if (!p)
3440			break;
3441	}
3442
3443	if (!p)
3444		return t_mod_start(m, pos);
3445
3446	return iter;
3447}
3448
3449static void t_stop(struct seq_file *m, void *p)
3450{
3451	mutex_unlock(&ftrace_lock);
3452}
3453
3454void * __weak
3455arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
3456{
3457	return NULL;
3458}
3459
3460static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
3461				struct dyn_ftrace *rec)
3462{
3463	void *ptr;
3464
3465	ptr = arch_ftrace_trampoline_func(ops, rec);
3466	if (ptr)
3467		seq_printf(m, " ->%pS", ptr);
3468}
3469
3470static int t_show(struct seq_file *m, void *v)
3471{
3472	struct ftrace_iterator *iter = m->private;
3473	struct dyn_ftrace *rec;
3474
3475	if (iter->flags & FTRACE_ITER_PROBE)
3476		return t_probe_show(m, iter);
3477
3478	if (iter->flags & FTRACE_ITER_MOD)
3479		return t_mod_show(m, iter);
3480
3481	if (iter->flags & FTRACE_ITER_PRINTALL) {
3482		if (iter->flags & FTRACE_ITER_NOTRACE)
3483			seq_puts(m, "#### no functions disabled ####\n");
3484		else
3485			seq_puts(m, "#### all functions enabled ####\n");
3486		return 0;
3487	}
3488
3489	rec = iter->func;
3490
3491	if (!rec)
3492		return 0;
3493
3494	seq_printf(m, "%ps", (void *)rec->ip);
3495	if (iter->flags & FTRACE_ITER_ENABLED) {
3496		struct ftrace_ops *ops;
3497
3498		seq_printf(m, " (%ld)%s%s",
3499			   ftrace_rec_count(rec),
3500			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
3501			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
 
3502		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3503			ops = ftrace_find_tramp_ops_any(rec);
3504			if (ops) {
3505				do {
3506					seq_printf(m, "\ttramp: %pS (%pS)",
3507						   (void *)ops->trampoline,
3508						   (void *)ops->func);
3509					add_trampoline_func(m, ops, rec);
3510					ops = ftrace_find_tramp_ops_next(rec, ops);
3511				} while (ops);
3512			} else
3513				seq_puts(m, "\ttramp: ERROR!");
3514		} else {
3515			add_trampoline_func(m, NULL, rec);
3516		}
3517	}	
 
 
 
 
 
 
 
3518
3519	seq_putc(m, '\n');
3520
3521	return 0;
3522}
3523
3524static const struct seq_operations show_ftrace_seq_ops = {
3525	.start = t_start,
3526	.next = t_next,
3527	.stop = t_stop,
3528	.show = t_show,
3529};
3530
3531static int
3532ftrace_avail_open(struct inode *inode, struct file *file)
3533{
3534	struct ftrace_iterator *iter;
 
 
 
 
 
3535
3536	if (unlikely(ftrace_disabled))
3537		return -ENODEV;
3538
3539	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3540	if (!iter)
3541		return -ENOMEM;
3542
3543	iter->pg = ftrace_pages_start;
3544	iter->ops = &global_ops;
3545
3546	return 0;
3547}
3548
3549static int
3550ftrace_enabled_open(struct inode *inode, struct file *file)
3551{
3552	struct ftrace_iterator *iter;
3553
 
 
 
 
 
 
 
 
 
3554	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3555	if (!iter)
3556		return -ENOMEM;
3557
3558	iter->pg = ftrace_pages_start;
3559	iter->flags = FTRACE_ITER_ENABLED;
3560	iter->ops = &global_ops;
3561
3562	return 0;
3563}
3564
3565/**
3566 * ftrace_regex_open - initialize function tracer filter files
3567 * @ops: The ftrace_ops that hold the hash filters
3568 * @flag: The type of filter to process
3569 * @inode: The inode, usually passed in to your open routine
3570 * @file: The file, usually passed in to your open routine
3571 *
3572 * ftrace_regex_open() initializes the filter files for the
3573 * @ops. Depending on @flag it may process the filter hash or
3574 * the notrace hash of @ops. With this called from the open
3575 * routine, you can use ftrace_filter_write() for the write
3576 * routine if @flag has FTRACE_ITER_FILTER set, or
3577 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3578 * tracing_lseek() should be used as the lseek routine, and
3579 * release must call ftrace_regex_release().
3580 */
3581int
3582ftrace_regex_open(struct ftrace_ops *ops, int flag,
3583		  struct inode *inode, struct file *file)
3584{
3585	struct ftrace_iterator *iter;
3586	struct ftrace_hash *hash;
3587	struct list_head *mod_head;
3588	struct trace_array *tr = ops->private;
3589	int ret = 0;
3590
3591	ftrace_ops_init(ops);
3592
3593	if (unlikely(ftrace_disabled))
3594		return -ENODEV;
3595
 
 
 
3596	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3597	if (!iter)
3598		return -ENOMEM;
3599
3600	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
3601		kfree(iter);
3602		return -ENOMEM;
3603	}
3604
3605	iter->ops = ops;
3606	iter->flags = flag;
3607	iter->tr = tr;
3608
3609	mutex_lock(&ops->func_hash->regex_lock);
3610
3611	if (flag & FTRACE_ITER_NOTRACE) {
3612		hash = ops->func_hash->notrace_hash;
3613		mod_head = tr ? &tr->mod_notrace : NULL;
3614	} else {
3615		hash = ops->func_hash->filter_hash;
3616		mod_head = tr ? &tr->mod_trace : NULL;
3617	}
3618
3619	iter->mod_list = mod_head;
3620
3621	if (file->f_mode & FMODE_WRITE) {
3622		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
3623
3624		if (file->f_flags & O_TRUNC) {
3625			iter->hash = alloc_ftrace_hash(size_bits);
3626			clear_ftrace_mod_list(mod_head);
3627	        } else {
3628			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
3629		}
3630
3631		if (!iter->hash) {
3632			trace_parser_put(&iter->parser);
3633			kfree(iter);
3634			ret = -ENOMEM;
3635			goto out_unlock;
3636		}
3637	} else
3638		iter->hash = hash;
3639
 
 
3640	if (file->f_mode & FMODE_READ) {
3641		iter->pg = ftrace_pages_start;
3642
3643		ret = seq_open(file, &show_ftrace_seq_ops);
3644		if (!ret) {
3645			struct seq_file *m = file->private_data;
3646			m->private = iter;
3647		} else {
3648			/* Failed */
3649			free_ftrace_hash(iter->hash);
3650			trace_parser_put(&iter->parser);
3651			kfree(iter);
3652		}
3653	} else
3654		file->private_data = iter;
3655
3656 out_unlock:
3657	mutex_unlock(&ops->func_hash->regex_lock);
3658
 
 
 
 
 
 
 
3659	return ret;
3660}
3661
3662static int
3663ftrace_filter_open(struct inode *inode, struct file *file)
3664{
3665	struct ftrace_ops *ops = inode->i_private;
3666
 
3667	return ftrace_regex_open(ops,
3668			FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
3669			inode, file);
3670}
3671
3672static int
3673ftrace_notrace_open(struct inode *inode, struct file *file)
3674{
3675	struct ftrace_ops *ops = inode->i_private;
3676
 
3677	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3678				 inode, file);
3679}
3680
3681/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
3682struct ftrace_glob {
3683	char *search;
3684	unsigned len;
3685	int type;
3686};
3687
3688/*
3689 * If symbols in an architecture don't correspond exactly to the user-visible
3690 * name of what they represent, it is possible to define this function to
3691 * perform the necessary adjustments.
3692*/
3693char * __weak arch_ftrace_match_adjust(char *str, const char *search)
3694{
3695	return str;
3696}
3697
3698static int ftrace_match(char *str, struct ftrace_glob *g)
3699{
3700	int matched = 0;
3701	int slen;
3702
3703	str = arch_ftrace_match_adjust(str, g->search);
3704
3705	switch (g->type) {
3706	case MATCH_FULL:
3707		if (strcmp(str, g->search) == 0)
3708			matched = 1;
3709		break;
3710	case MATCH_FRONT_ONLY:
3711		if (strncmp(str, g->search, g->len) == 0)
3712			matched = 1;
3713		break;
3714	case MATCH_MIDDLE_ONLY:
3715		if (strstr(str, g->search))
3716			matched = 1;
3717		break;
3718	case MATCH_END_ONLY:
3719		slen = strlen(str);
3720		if (slen >= g->len &&
3721		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3722			matched = 1;
3723		break;
3724	case MATCH_GLOB:
3725		if (glob_match(g->search, str))
3726			matched = 1;
3727		break;
3728	}
3729
3730	return matched;
3731}
3732
3733static int
3734enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3735{
3736	struct ftrace_func_entry *entry;
3737	int ret = 0;
3738
3739	entry = ftrace_lookup_ip(hash, rec->ip);
3740	if (clear_filter) {
3741		/* Do nothing if it doesn't exist */
3742		if (!entry)
3743			return 0;
3744
3745		free_hash_entry(hash, entry);
3746	} else {
3747		/* Do nothing if it exists */
3748		if (entry)
3749			return 0;
3750
3751		ret = add_hash_entry(hash, rec->ip);
3752	}
3753	return ret;
3754}
3755
3756static int
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3757ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
3758		struct ftrace_glob *mod_g, int exclude_mod)
3759{
3760	char str[KSYM_SYMBOL_LEN];
3761	char *modname;
3762
3763	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
3764
3765	if (mod_g) {
3766		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
3767
3768		/* blank module name to match all modules */
3769		if (!mod_g->len) {
3770			/* blank module globbing: modname xor exclude_mod */
3771			if (!exclude_mod != !modname)
3772				goto func_match;
3773			return 0;
3774		}
3775
3776		/*
3777		 * exclude_mod is set to trace everything but the given
3778		 * module. If it is set and the module matches, then
3779		 * return 0. If it is not set, and the module doesn't match
3780		 * also return 0. Otherwise, check the function to see if
3781		 * that matches.
3782		 */
3783		if (!mod_matches == !exclude_mod)
3784			return 0;
3785func_match:
3786		/* blank search means to match all funcs in the mod */
3787		if (!func_g->len)
3788			return 1;
3789	}
3790
3791	return ftrace_match(str, func_g);
3792}
3793
3794static int
3795match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3796{
3797	struct ftrace_page *pg;
3798	struct dyn_ftrace *rec;
3799	struct ftrace_glob func_g = { .type = MATCH_FULL };
3800	struct ftrace_glob mod_g = { .type = MATCH_FULL };
3801	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
3802	int exclude_mod = 0;
3803	int found = 0;
3804	int ret;
3805	int clear_filter = 0;
3806
3807	if (func) {
3808		func_g.type = filter_parse_regex(func, len, &func_g.search,
3809						 &clear_filter);
3810		func_g.len = strlen(func_g.search);
3811	}
3812
3813	if (mod) {
3814		mod_g.type = filter_parse_regex(mod, strlen(mod),
3815				&mod_g.search, &exclude_mod);
3816		mod_g.len = strlen(mod_g.search);
3817	}
3818
3819	mutex_lock(&ftrace_lock);
3820
3821	if (unlikely(ftrace_disabled))
3822		goto out_unlock;
3823
 
 
 
 
 
3824	do_for_each_ftrace_rec(pg, rec) {
3825
3826		if (rec->flags & FTRACE_FL_DISABLED)
3827			continue;
3828
3829		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3830			ret = enter_record(hash, rec, clear_filter);
3831			if (ret < 0) {
3832				found = ret;
3833				goto out_unlock;
3834			}
3835			found = 1;
3836		}
3837	} while_for_each_ftrace_rec();
3838 out_unlock:
3839	mutex_unlock(&ftrace_lock);
3840
3841	return found;
3842}
3843
3844static int
3845ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3846{
3847	return match_records(hash, buff, len, NULL);
3848}
3849
3850static void ftrace_ops_update_code(struct ftrace_ops *ops,
3851				   struct ftrace_ops_hash *old_hash)
3852{
3853	struct ftrace_ops *op;
3854
3855	if (!ftrace_enabled)
3856		return;
3857
3858	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
3859		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
3860		return;
3861	}
3862
3863	/*
3864	 * If this is the shared global_ops filter, then we need to
3865	 * check if there is another ops that shares it, is enabled.
3866	 * If so, we still need to run the modify code.
3867	 */
3868	if (ops->func_hash != &global_ops.local_hash)
3869		return;
3870
3871	do_for_each_ftrace_op(op, ftrace_ops_list) {
3872		if (op->func_hash == &global_ops.local_hash &&
3873		    op->flags & FTRACE_OPS_FL_ENABLED) {
3874			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
3875			/* Only need to do this once */
3876			return;
3877		}
3878	} while_for_each_ftrace_op(op);
3879}
3880
3881static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
3882					   struct ftrace_hash **orig_hash,
3883					   struct ftrace_hash *hash,
3884					   int enable)
3885{
3886	struct ftrace_ops_hash old_hash_ops;
3887	struct ftrace_hash *old_hash;
3888	int ret;
3889
3890	old_hash = *orig_hash;
3891	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
3892	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
3893	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3894	if (!ret) {
3895		ftrace_ops_update_code(ops, &old_hash_ops);
3896		free_ftrace_hash_rcu(old_hash);
3897	}
3898	return ret;
3899}
3900
3901static bool module_exists(const char *module)
3902{
3903	/* All modules have the symbol __this_module */
3904	const char this_mod[] = "__this_module";
3905	char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2];
3906	unsigned long val;
3907	int n;
3908
3909	n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
3910
3911	if (n > sizeof(modname) - 1)
3912		return false;
3913
3914	val = module_kallsyms_lookup_name(modname);
3915	return val != 0;
3916}
3917
3918static int cache_mod(struct trace_array *tr,
3919		     const char *func, char *module, int enable)
3920{
3921	struct ftrace_mod_load *ftrace_mod, *n;
3922	struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
3923	int ret;
3924
3925	mutex_lock(&ftrace_lock);
3926
3927	/* We do not cache inverse filters */
3928	if (func[0] == '!') {
3929		func++;
3930		ret = -EINVAL;
3931
3932		/* Look to remove this hash */
3933		list_for_each_entry_safe(ftrace_mod, n, head, list) {
3934			if (strcmp(ftrace_mod->module, module) != 0)
3935				continue;
3936
3937			/* no func matches all */
3938			if (strcmp(func, "*") == 0 ||
3939			    (ftrace_mod->func &&
3940			     strcmp(ftrace_mod->func, func) == 0)) {
3941				ret = 0;
3942				free_ftrace_mod(ftrace_mod);
3943				continue;
3944			}
3945		}
3946		goto out;
3947	}
3948
3949	ret = -EINVAL;
3950	/* We only care about modules that have not been loaded yet */
3951	if (module_exists(module))
3952		goto out;
3953
3954	/* Save this string off, and execute it when the module is loaded */
3955	ret = ftrace_add_mod(tr, func, module, enable);
3956 out:
3957	mutex_unlock(&ftrace_lock);
3958
3959	return ret;
3960}
3961
3962static int
3963ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
3964		 int reset, int enable);
3965
3966#ifdef CONFIG_MODULES
3967static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
3968			     char *mod, bool enable)
3969{
3970	struct ftrace_mod_load *ftrace_mod, *n;
3971	struct ftrace_hash **orig_hash, *new_hash;
3972	LIST_HEAD(process_mods);
3973	char *func;
3974	int ret;
3975
3976	mutex_lock(&ops->func_hash->regex_lock);
3977
3978	if (enable)
3979		orig_hash = &ops->func_hash->filter_hash;
3980	else
3981		orig_hash = &ops->func_hash->notrace_hash;
3982
3983	new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
3984					      *orig_hash);
3985	if (!new_hash)
3986		goto out; /* warn? */
3987
3988	mutex_lock(&ftrace_lock);
3989
3990	list_for_each_entry_safe(ftrace_mod, n, head, list) {
3991
3992		if (strcmp(ftrace_mod->module, mod) != 0)
3993			continue;
3994
3995		if (ftrace_mod->func)
3996			func = kstrdup(ftrace_mod->func, GFP_KERNEL);
3997		else
3998			func = kstrdup("*", GFP_KERNEL);
3999
4000		if (!func) /* warn? */
4001			continue;
4002
4003		list_del(&ftrace_mod->list);
4004		list_add(&ftrace_mod->list, &process_mods);
4005
4006		/* Use the newly allocated func, as it may be "*" */
4007		kfree(ftrace_mod->func);
4008		ftrace_mod->func = func;
4009	}
4010
4011	mutex_unlock(&ftrace_lock);
4012
4013	list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {
4014
4015		func = ftrace_mod->func;
4016
4017		/* Grabs ftrace_lock, which is why we have this extra step */
4018		match_records(new_hash, func, strlen(func), mod);
4019		free_ftrace_mod(ftrace_mod);
4020	}
4021
4022	if (enable && list_empty(head))
4023		new_hash->flags &= ~FTRACE_HASH_FL_MOD;
4024
4025	mutex_lock(&ftrace_lock);
4026
4027	ret = ftrace_hash_move_and_update_ops(ops, orig_hash,
4028					      new_hash, enable);
4029	mutex_unlock(&ftrace_lock);
4030
4031 out:
4032	mutex_unlock(&ops->func_hash->regex_lock);
4033
4034	free_ftrace_hash(new_hash);
4035}
4036
4037static void process_cached_mods(const char *mod_name)
4038{
4039	struct trace_array *tr;
4040	char *mod;
4041
4042	mod = kstrdup(mod_name, GFP_KERNEL);
4043	if (!mod)
4044		return;
4045
4046	mutex_lock(&trace_types_lock);
4047	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
4048		if (!list_empty(&tr->mod_trace))
4049			process_mod_list(&tr->mod_trace, tr->ops, mod, true);
4050		if (!list_empty(&tr->mod_notrace))
4051			process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
4052	}
4053	mutex_unlock(&trace_types_lock);
4054
4055	kfree(mod);
4056}
4057#endif
4058
4059/*
4060 * We register the module command as a template to show others how
4061 * to register the a command as well.
4062 */
4063
4064static int
4065ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
4066		    char *func_orig, char *cmd, char *module, int enable)
4067{
4068	char *func;
4069	int ret;
4070
4071	/* match_records() modifies func, and we need the original */
4072	func = kstrdup(func_orig, GFP_KERNEL);
4073	if (!func)
4074		return -ENOMEM;
4075
4076	/*
4077	 * cmd == 'mod' because we only registered this func
4078	 * for the 'mod' ftrace_func_command.
4079	 * But if you register one func with multiple commands,
4080	 * you can tell which command was used by the cmd
4081	 * parameter.
4082	 */
4083	ret = match_records(hash, func, strlen(func), module);
4084	kfree(func);
4085
4086	if (!ret)
4087		return cache_mod(tr, func_orig, module, enable);
4088	if (ret < 0)
4089		return ret;
4090	return 0;
4091}
4092
4093static struct ftrace_func_command ftrace_mod_cmd = {
4094	.name			= "mod",
4095	.func			= ftrace_mod_callback,
4096};
4097
4098static int __init ftrace_mod_cmd_init(void)
4099{
4100	return register_ftrace_command(&ftrace_mod_cmd);
4101}
4102core_initcall(ftrace_mod_cmd_init);
4103
4104static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
4105				      struct ftrace_ops *op, struct pt_regs *pt_regs)
4106{
4107	struct ftrace_probe_ops *probe_ops;
4108	struct ftrace_func_probe *probe;
4109
4110	probe = container_of(op, struct ftrace_func_probe, ops);
4111	probe_ops = probe->probe_ops;
4112
4113	/*
4114	 * Disable preemption for these calls to prevent a RCU grace
4115	 * period. This syncs the hash iteration and freeing of items
4116	 * on the hash. rcu_read_lock is too dangerous here.
4117	 */
4118	preempt_disable_notrace();
4119	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
4120	preempt_enable_notrace();
4121}
4122
4123struct ftrace_func_map {
4124	struct ftrace_func_entry	entry;
4125	void				*data;
4126};
4127
4128struct ftrace_func_mapper {
4129	struct ftrace_hash		hash;
4130};
4131
4132/**
4133 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
4134 *
4135 * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
4136 */
4137struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
4138{
4139	struct ftrace_hash *hash;
4140
4141	/*
4142	 * The mapper is simply a ftrace_hash, but since the entries
4143	 * in the hash are not ftrace_func_entry type, we define it
4144	 * as a separate structure.
4145	 */
4146	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4147	return (struct ftrace_func_mapper *)hash;
4148}
4149
4150/**
4151 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
4152 * @mapper: The mapper that has the ip maps
4153 * @ip: the instruction pointer to find the data for
4154 *
4155 * Returns the data mapped to @ip if found otherwise NULL. The return
4156 * is actually the address of the mapper data pointer. The address is
4157 * returned for use cases where the data is no bigger than a long, and
4158 * the user can use the data pointer as its data instead of having to
4159 * allocate more memory for the reference.
4160 */
4161void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
4162				  unsigned long ip)
4163{
4164	struct ftrace_func_entry *entry;
4165	struct ftrace_func_map *map;
4166
4167	entry = ftrace_lookup_ip(&mapper->hash, ip);
4168	if (!entry)
4169		return NULL;
4170
4171	map = (struct ftrace_func_map *)entry;
4172	return &map->data;
4173}
4174
4175/**
4176 * ftrace_func_mapper_add_ip - Map some data to an ip
4177 * @mapper: The mapper that has the ip maps
4178 * @ip: The instruction pointer address to map @data to
4179 * @data: The data to map to @ip
4180 *
4181 * Returns 0 on succes otherwise an error.
4182 */
4183int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
4184			      unsigned long ip, void *data)
4185{
4186	struct ftrace_func_entry *entry;
4187	struct ftrace_func_map *map;
4188
4189	entry = ftrace_lookup_ip(&mapper->hash, ip);
4190	if (entry)
4191		return -EBUSY;
4192
4193	map = kmalloc(sizeof(*map), GFP_KERNEL);
4194	if (!map)
4195		return -ENOMEM;
4196
4197	map->entry.ip = ip;
4198	map->data = data;
4199
4200	__add_hash_entry(&mapper->hash, &map->entry);
4201
4202	return 0;
4203}
4204
4205/**
4206 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
4207 * @mapper: The mapper that has the ip maps
4208 * @ip: The instruction pointer address to remove the data from
4209 *
4210 * Returns the data if it is found, otherwise NULL.
4211 * Note, if the data pointer is used as the data itself, (see 
4212 * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
4213 * if the data pointer was set to zero.
4214 */
4215void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
4216				   unsigned long ip)
4217{
4218	struct ftrace_func_entry *entry;
4219	struct ftrace_func_map *map;
4220	void *data;
4221
4222	entry = ftrace_lookup_ip(&mapper->hash, ip);
4223	if (!entry)
4224		return NULL;
4225
4226	map = (struct ftrace_func_map *)entry;
4227	data = map->data;
4228
4229	remove_hash_entry(&mapper->hash, entry);
4230	kfree(entry);
4231
4232	return data;
4233}
4234
4235/**
4236 * free_ftrace_func_mapper - free a mapping of ips and data
4237 * @mapper: The mapper that has the ip maps
4238 * @free_func: A function to be called on each data item.
4239 *
4240 * This is used to free the function mapper. The @free_func is optional
4241 * and can be used if the data needs to be freed as well.
4242 */
4243void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
4244			     ftrace_mapper_func free_func)
4245{
4246	struct ftrace_func_entry *entry;
4247	struct ftrace_func_map *map;
4248	struct hlist_head *hhd;
4249	int size = 1 << mapper->hash.size_bits;
4250	int i;
 
 
4251
4252	if (free_func && mapper->hash.count) {
 
4253		for (i = 0; i < size; i++) {
4254			hhd = &mapper->hash.buckets[i];
4255			hlist_for_each_entry(entry, hhd, hlist) {
4256				map = (struct ftrace_func_map *)entry;
4257				free_func(map);
4258			}
4259		}
4260	}
4261	free_ftrace_hash(&mapper->hash);
4262}
4263
4264static void release_probe(struct ftrace_func_probe *probe)
4265{
4266	struct ftrace_probe_ops *probe_ops;
4267
4268	mutex_lock(&ftrace_lock);
4269
4270	WARN_ON(probe->ref <= 0);
4271
4272	/* Subtract the ref that was used to protect this instance */
4273	probe->ref--;
4274
4275	if (!probe->ref) {
4276		probe_ops = probe->probe_ops;
4277		/*
4278		 * Sending zero as ip tells probe_ops to free
4279		 * the probe->data itself
4280		 */
4281		if (probe_ops->free)
4282			probe_ops->free(probe_ops, probe->tr, 0, probe->data);
4283		list_del(&probe->list);
4284		kfree(probe);
4285	}
4286	mutex_unlock(&ftrace_lock);
4287}
4288
4289static void acquire_probe_locked(struct ftrace_func_probe *probe)
4290{
4291	/*
4292	 * Add one ref to keep it from being freed when releasing the
4293	 * ftrace_lock mutex.
4294	 */
4295	probe->ref++;
4296}
4297
4298int
4299register_ftrace_function_probe(char *glob, struct trace_array *tr,
4300			       struct ftrace_probe_ops *probe_ops,
4301			       void *data)
4302{
4303	struct ftrace_func_entry *entry;
4304	struct ftrace_func_probe *probe;
4305	struct ftrace_hash **orig_hash;
4306	struct ftrace_hash *old_hash;
4307	struct ftrace_hash *hash;
4308	int count = 0;
4309	int size;
4310	int ret;
4311	int i;
4312
4313	if (WARN_ON(!tr))
4314		return -EINVAL;
4315
4316	/* We do not support '!' for function probes */
4317	if (WARN_ON(glob[0] == '!'))
4318		return -EINVAL;
4319
4320
4321	mutex_lock(&ftrace_lock);
4322	/* Check if the probe_ops is already registered */
4323	list_for_each_entry(probe, &tr->func_probes, list) {
4324		if (probe->probe_ops == probe_ops)
4325			break;
4326	}
4327	if (&probe->list == &tr->func_probes) {
4328		probe = kzalloc(sizeof(*probe), GFP_KERNEL);
4329		if (!probe) {
4330			mutex_unlock(&ftrace_lock);
4331			return -ENOMEM;
4332		}
4333		probe->probe_ops = probe_ops;
4334		probe->ops.func = function_trace_probe_call;
4335		probe->tr = tr;
4336		ftrace_ops_init(&probe->ops);
4337		list_add(&probe->list, &tr->func_probes);
4338	}
4339
4340	acquire_probe_locked(probe);
4341
4342	mutex_unlock(&ftrace_lock);
4343
 
 
 
 
4344	mutex_lock(&probe->ops.func_hash->regex_lock);
4345
4346	orig_hash = &probe->ops.func_hash->filter_hash;
4347	old_hash = *orig_hash;
4348	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4349
 
 
 
 
 
4350	ret = ftrace_match_records(hash, glob, strlen(glob));
4351
4352	/* Nothing found? */
4353	if (!ret)
4354		ret = -EINVAL;
4355
4356	if (ret < 0)
4357		goto out;
4358
4359	size = 1 << hash->size_bits;
4360	for (i = 0; i < size; i++) {
4361		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
4362			if (ftrace_lookup_ip(old_hash, entry->ip))
4363				continue;
4364			/*
4365			 * The caller might want to do something special
4366			 * for each function we find. We call the callback
4367			 * to give the caller an opportunity to do so.
4368			 */
4369			if (probe_ops->init) {
4370				ret = probe_ops->init(probe_ops, tr,
4371						      entry->ip, data,
4372						      &probe->data);
4373				if (ret < 0) {
4374					if (probe_ops->free && count)
4375						probe_ops->free(probe_ops, tr,
4376								0, probe->data);
4377					probe->data = NULL;
4378					goto out;
4379				}
4380			}
4381			count++;
4382		}
4383	}
4384
4385	mutex_lock(&ftrace_lock);
4386
4387	if (!count) {
4388		/* Nothing was added? */
4389		ret = -EINVAL;
4390		goto out_unlock;
4391	}
4392
4393	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4394					      hash, 1);
4395	if (ret < 0)
4396		goto err_unlock;
4397
4398	/* One ref for each new function traced */
4399	probe->ref += count;
4400
4401	if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
4402		ret = ftrace_startup(&probe->ops, 0);
4403
4404 out_unlock:
4405	mutex_unlock(&ftrace_lock);
4406
4407	if (!ret)
4408		ret = count;
4409 out:
4410	mutex_unlock(&probe->ops.func_hash->regex_lock);
4411	free_ftrace_hash(hash);
4412
4413	release_probe(probe);
4414
4415	return ret;
4416
4417 err_unlock:
4418	if (!probe_ops->free || !count)
4419		goto out_unlock;
4420
4421	/* Failed to do the move, need to call the free functions */
4422	for (i = 0; i < size; i++) {
4423		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
4424			if (ftrace_lookup_ip(old_hash, entry->ip))
4425				continue;
4426			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4427		}
4428	}
4429	goto out_unlock;
4430}
4431
4432int
4433unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
4434				      struct ftrace_probe_ops *probe_ops)
4435{
4436	struct ftrace_ops_hash old_hash_ops;
4437	struct ftrace_func_entry *entry;
4438	struct ftrace_func_probe *probe;
4439	struct ftrace_glob func_g;
4440	struct ftrace_hash **orig_hash;
4441	struct ftrace_hash *old_hash;
4442	struct ftrace_hash *hash = NULL;
4443	struct hlist_node *tmp;
4444	struct hlist_head hhd;
4445	char str[KSYM_SYMBOL_LEN];
4446	int count = 0;
4447	int i, ret = -ENODEV;
4448	int size;
4449
4450	if (!glob || !strlen(glob) || !strcmp(glob, "*"))
4451		func_g.search = NULL;
4452	else {
4453		int not;
4454
4455		func_g.type = filter_parse_regex(glob, strlen(glob),
4456						 &func_g.search, &not);
4457		func_g.len = strlen(func_g.search);
4458
4459		/* we do not support '!' for function probes */
4460		if (WARN_ON(not))
4461			return -EINVAL;
4462	}
4463
4464	mutex_lock(&ftrace_lock);
4465	/* Check if the probe_ops is already registered */
4466	list_for_each_entry(probe, &tr->func_probes, list) {
4467		if (probe->probe_ops == probe_ops)
4468			break;
4469	}
4470	if (&probe->list == &tr->func_probes)
4471		goto err_unlock_ftrace;
4472
4473	ret = -EINVAL;
4474	if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
4475		goto err_unlock_ftrace;
4476
4477	acquire_probe_locked(probe);
4478
4479	mutex_unlock(&ftrace_lock);
4480
4481	mutex_lock(&probe->ops.func_hash->regex_lock);
4482
4483	orig_hash = &probe->ops.func_hash->filter_hash;
4484	old_hash = *orig_hash;
4485
4486	if (ftrace_hash_empty(old_hash))
4487		goto out_unlock;
4488
4489	old_hash_ops.filter_hash = old_hash;
4490	/* Probes only have filters */
4491	old_hash_ops.notrace_hash = NULL;
4492
4493	ret = -ENOMEM;
4494	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4495	if (!hash)
4496		goto out_unlock;
4497
4498	INIT_HLIST_HEAD(&hhd);
4499
4500	size = 1 << hash->size_bits;
4501	for (i = 0; i < size; i++) {
4502		hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
4503
4504			if (func_g.search) {
4505				kallsyms_lookup(entry->ip, NULL, NULL,
4506						NULL, str);
4507				if (!ftrace_match(str, &func_g))
4508					continue;
4509			}
4510			count++;
4511			remove_hash_entry(hash, entry);
4512			hlist_add_head(&entry->hlist, &hhd);
4513		}
4514	}
4515
4516	/* Nothing found? */
4517	if (!count) {
4518		ret = -EINVAL;
4519		goto out_unlock;
4520	}
4521
4522	mutex_lock(&ftrace_lock);
4523
4524	WARN_ON(probe->ref < count);
4525
4526	probe->ref -= count;
4527
4528	if (ftrace_hash_empty(hash))
4529		ftrace_shutdown(&probe->ops, 0);
4530
4531	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4532					      hash, 1);
4533
4534	/* still need to update the function call sites */
4535	if (ftrace_enabled && !ftrace_hash_empty(hash))
4536		ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
4537				       &old_hash_ops);
4538	synchronize_sched();
4539
4540	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
4541		hlist_del(&entry->hlist);
4542		if (probe_ops->free)
4543			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4544		kfree(entry);
4545	}
4546	mutex_unlock(&ftrace_lock);
4547
4548 out_unlock:
4549	mutex_unlock(&probe->ops.func_hash->regex_lock);
4550	free_ftrace_hash(hash);
4551
4552	release_probe(probe);
4553
4554	return ret;
4555
4556 err_unlock_ftrace:
4557	mutex_unlock(&ftrace_lock);
4558	return ret;
4559}
4560
4561void clear_ftrace_function_probes(struct trace_array *tr)
4562{
4563	struct ftrace_func_probe *probe, *n;
4564
4565	list_for_each_entry_safe(probe, n, &tr->func_probes, list)
4566		unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
4567}
4568
4569static LIST_HEAD(ftrace_commands);
4570static DEFINE_MUTEX(ftrace_cmd_mutex);
4571
4572/*
4573 * Currently we only register ftrace commands from __init, so mark this
4574 * __init too.
4575 */
4576__init int register_ftrace_command(struct ftrace_func_command *cmd)
4577{
4578	struct ftrace_func_command *p;
4579	int ret = 0;
4580
4581	mutex_lock(&ftrace_cmd_mutex);
4582	list_for_each_entry(p, &ftrace_commands, list) {
4583		if (strcmp(cmd->name, p->name) == 0) {
4584			ret = -EBUSY;
4585			goto out_unlock;
4586		}
4587	}
4588	list_add(&cmd->list, &ftrace_commands);
4589 out_unlock:
4590	mutex_unlock(&ftrace_cmd_mutex);
4591
4592	return ret;
4593}
4594
4595/*
4596 * Currently we only unregister ftrace commands from __init, so mark
4597 * this __init too.
4598 */
4599__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4600{
4601	struct ftrace_func_command *p, *n;
4602	int ret = -ENODEV;
4603
4604	mutex_lock(&ftrace_cmd_mutex);
4605	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
4606		if (strcmp(cmd->name, p->name) == 0) {
4607			ret = 0;
4608			list_del_init(&p->list);
4609			goto out_unlock;
4610		}
4611	}
4612 out_unlock:
4613	mutex_unlock(&ftrace_cmd_mutex);
4614
4615	return ret;
4616}
4617
4618static int ftrace_process_regex(struct ftrace_iterator *iter,
4619				char *buff, int len, int enable)
4620{
4621	struct ftrace_hash *hash = iter->hash;
4622	struct trace_array *tr = iter->ops->private;
4623	char *func, *command, *next = buff;
4624	struct ftrace_func_command *p;
4625	int ret = -EINVAL;
4626
4627	func = strsep(&next, ":");
4628
4629	if (!next) {
4630		ret = ftrace_match_records(hash, func, len);
4631		if (!ret)
4632			ret = -EINVAL;
4633		if (ret < 0)
4634			return ret;
4635		return 0;
4636	}
4637
4638	/* command found */
4639
4640	command = strsep(&next, ":");
4641
4642	mutex_lock(&ftrace_cmd_mutex);
4643	list_for_each_entry(p, &ftrace_commands, list) {
4644		if (strcmp(p->name, command) == 0) {
4645			ret = p->func(tr, hash, func, command, next, enable);
4646			goto out_unlock;
4647		}
4648	}
4649 out_unlock:
4650	mutex_unlock(&ftrace_cmd_mutex);
4651
4652	return ret;
4653}
4654
4655static ssize_t
4656ftrace_regex_write(struct file *file, const char __user *ubuf,
4657		   size_t cnt, loff_t *ppos, int enable)
4658{
4659	struct ftrace_iterator *iter;
4660	struct trace_parser *parser;
4661	ssize_t ret, read;
4662
4663	if (!cnt)
4664		return 0;
4665
4666	if (file->f_mode & FMODE_READ) {
4667		struct seq_file *m = file->private_data;
4668		iter = m->private;
4669	} else
4670		iter = file->private_data;
4671
4672	if (unlikely(ftrace_disabled))
4673		return -ENODEV;
4674
4675	/* iter->hash is a local copy, so we don't need regex_lock */
4676
4677	parser = &iter->parser;
4678	read = trace_get_user(parser, ubuf, cnt, ppos);
4679
4680	if (read >= 0 && trace_parser_loaded(parser) &&
4681	    !trace_parser_cont(parser)) {
4682		ret = ftrace_process_regex(iter, parser->buffer,
4683					   parser->idx, enable);
4684		trace_parser_clear(parser);
4685		if (ret < 0)
4686			goto out;
4687	}
4688
4689	ret = read;
4690 out:
4691	return ret;
4692}
4693
4694ssize_t
4695ftrace_filter_write(struct file *file, const char __user *ubuf,
4696		    size_t cnt, loff_t *ppos)
4697{
4698	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
4699}
4700
4701ssize_t
4702ftrace_notrace_write(struct file *file, const char __user *ubuf,
4703		     size_t cnt, loff_t *ppos)
4704{
4705	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
4706}
4707
4708static int
4709ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
4710{
4711	struct ftrace_func_entry *entry;
4712
4713	if (!ftrace_location(ip))
4714		return -EINVAL;
4715
4716	if (remove) {
4717		entry = ftrace_lookup_ip(hash, ip);
4718		if (!entry)
4719			return -ENOENT;
4720		free_hash_entry(hash, entry);
4721		return 0;
4722	}
4723
4724	return add_hash_entry(hash, ip);
4725}
4726
4727static int
4728ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
4729		unsigned long ip, int remove, int reset, int enable)
4730{
4731	struct ftrace_hash **orig_hash;
4732	struct ftrace_hash *hash;
4733	int ret;
4734
4735	if (unlikely(ftrace_disabled))
4736		return -ENODEV;
4737
4738	mutex_lock(&ops->func_hash->regex_lock);
4739
4740	if (enable)
4741		orig_hash = &ops->func_hash->filter_hash;
4742	else
4743		orig_hash = &ops->func_hash->notrace_hash;
4744
4745	if (reset)
4746		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4747	else
4748		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
4749
4750	if (!hash) {
4751		ret = -ENOMEM;
4752		goto out_regex_unlock;
4753	}
4754
4755	if (buf && !ftrace_match_records(hash, buf, len)) {
4756		ret = -EINVAL;
4757		goto out_regex_unlock;
4758	}
4759	if (ip) {
4760		ret = ftrace_match_addr(hash, ip, remove);
4761		if (ret < 0)
4762			goto out_regex_unlock;
4763	}
4764
4765	mutex_lock(&ftrace_lock);
4766	ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
4767	mutex_unlock(&ftrace_lock);
4768
4769 out_regex_unlock:
4770	mutex_unlock(&ops->func_hash->regex_lock);
4771
4772	free_ftrace_hash(hash);
4773	return ret;
4774}
4775
4776static int
4777ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
4778		int reset, int enable)
4779{
4780	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4781}
4782
4783/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4784 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
4785 * @ops - the ops to set the filter with
4786 * @ip - the address to add to or remove from the filter.
4787 * @remove - non zero to remove the ip from the filter
4788 * @reset - non zero to reset all filters before applying this filter.
4789 *
4790 * Filters denote which functions should be enabled when tracing is enabled
4791 * If @ip is NULL, it failes to update filter.
4792 */
4793int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
4794			 int remove, int reset)
4795{
4796	ftrace_ops_init(ops);
4797	return ftrace_set_addr(ops, ip, remove, reset, 1);
4798}
4799EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
4800
4801/**
4802 * ftrace_ops_set_global_filter - setup ops to use global filters
4803 * @ops - the ops which will use the global filters
4804 *
4805 * ftrace users who need global function trace filtering should call this.
4806 * It can set the global filter only if ops were not initialized before.
4807 */
4808void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
4809{
4810	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
4811		return;
4812
4813	ftrace_ops_init(ops);
4814	ops->func_hash = &global_ops.local_hash;
4815}
4816EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
4817
4818static int
4819ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
4820		 int reset, int enable)
4821{
4822	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
4823}
4824
4825/**
4826 * ftrace_set_filter - set a function to filter on in ftrace
4827 * @ops - the ops to set the filter with
4828 * @buf - the string that holds the function filter text.
4829 * @len - the length of the string.
4830 * @reset - non zero to reset all filters before applying this filter.
4831 *
4832 * Filters denote which functions should be enabled when tracing is enabled.
4833 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
4834 */
4835int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4836		       int len, int reset)
4837{
4838	ftrace_ops_init(ops);
4839	return ftrace_set_regex(ops, buf, len, reset, 1);
4840}
4841EXPORT_SYMBOL_GPL(ftrace_set_filter);
4842
4843/**
4844 * ftrace_set_notrace - set a function to not trace in ftrace
4845 * @ops - the ops to set the notrace filter with
4846 * @buf - the string that holds the function notrace text.
4847 * @len - the length of the string.
4848 * @reset - non zero to reset all filters before applying this filter.
4849 *
4850 * Notrace Filters denote which functions should not be enabled when tracing
4851 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
4852 * for tracing.
4853 */
4854int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4855			int len, int reset)
4856{
4857	ftrace_ops_init(ops);
4858	return ftrace_set_regex(ops, buf, len, reset, 0);
4859}
4860EXPORT_SYMBOL_GPL(ftrace_set_notrace);
4861/**
4862 * ftrace_set_global_filter - set a function to filter on with global tracers
4863 * @buf - the string that holds the function filter text.
4864 * @len - the length of the string.
4865 * @reset - non zero to reset all filters before applying this filter.
4866 *
4867 * Filters denote which functions should be enabled when tracing is enabled.
4868 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
4869 */
4870void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4871{
4872	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4873}
4874EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4875
4876/**
4877 * ftrace_set_global_notrace - set a function to not trace with global tracers
4878 * @buf - the string that holds the function notrace text.
4879 * @len - the length of the string.
4880 * @reset - non zero to reset all filters before applying this filter.
4881 *
4882 * Notrace Filters denote which functions should not be enabled when tracing
4883 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
4884 * for tracing.
4885 */
4886void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4887{
4888	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4889}
4890EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4891
4892/*
4893 * command line interface to allow users to set filters on boot up.
4894 */
4895#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
4896static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4897static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
4898
4899/* Used by function selftest to not test if filter is set */
4900bool ftrace_filter_param __initdata;
4901
4902static int __init set_ftrace_notrace(char *str)
4903{
4904	ftrace_filter_param = true;
4905	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4906	return 1;
4907}
4908__setup("ftrace_notrace=", set_ftrace_notrace);
4909
4910static int __init set_ftrace_filter(char *str)
4911{
4912	ftrace_filter_param = true;
4913	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4914	return 1;
4915}
4916__setup("ftrace_filter=", set_ftrace_filter);
4917
4918#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4919static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4920static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4921static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
4922
4923static int __init set_graph_function(char *str)
4924{
4925	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4926	return 1;
4927}
4928__setup("ftrace_graph_filter=", set_graph_function);
4929
4930static int __init set_graph_notrace_function(char *str)
4931{
4932	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
4933	return 1;
4934}
4935__setup("ftrace_graph_notrace=", set_graph_notrace_function);
4936
4937static int __init set_graph_max_depth_function(char *str)
4938{
4939	if (!str)
4940		return 0;
4941	fgraph_max_depth = simple_strtoul(str, NULL, 0);
4942	return 1;
4943}
4944__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
4945
4946static void __init set_ftrace_early_graph(char *buf, int enable)
4947{
4948	int ret;
4949	char *func;
4950	struct ftrace_hash *hash;
4951
4952	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4953	if (WARN_ON(!hash))
4954		return;
4955
4956	while (buf) {
4957		func = strsep(&buf, ",");
4958		/* we allow only one expression at a time */
4959		ret = ftrace_graph_set_hash(hash, func);
4960		if (ret)
4961			printk(KERN_DEBUG "ftrace: function %s not "
4962					  "traceable\n", func);
4963	}
4964
4965	if (enable)
4966		ftrace_graph_hash = hash;
4967	else
4968		ftrace_graph_notrace_hash = hash;
4969}
4970#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4971
4972void __init
4973ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
4974{
4975	char *func;
4976
4977	ftrace_ops_init(ops);
4978
4979	while (buf) {
4980		func = strsep(&buf, ",");
4981		ftrace_set_regex(ops, func, strlen(func), 0, enable);
4982	}
4983}
4984
4985static void __init set_ftrace_early_filters(void)
4986{
4987	if (ftrace_filter_buf[0])
4988		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
4989	if (ftrace_notrace_buf[0])
4990		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
4991#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4992	if (ftrace_graph_buf[0])
4993		set_ftrace_early_graph(ftrace_graph_buf, 1);
4994	if (ftrace_graph_notrace_buf[0])
4995		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
4996#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4997}
4998
4999int ftrace_regex_release(struct inode *inode, struct file *file)
5000{
5001	struct seq_file *m = (struct seq_file *)file->private_data;
5002	struct ftrace_iterator *iter;
5003	struct ftrace_hash **orig_hash;
5004	struct trace_parser *parser;
5005	int filter_hash;
5006	int ret;
5007
5008	if (file->f_mode & FMODE_READ) {
5009		iter = m->private;
5010		seq_release(inode, file);
5011	} else
5012		iter = file->private_data;
5013
5014	parser = &iter->parser;
5015	if (trace_parser_loaded(parser)) {
5016		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
5017	}
5018
5019	trace_parser_put(parser);
5020
5021	mutex_lock(&iter->ops->func_hash->regex_lock);
5022
5023	if (file->f_mode & FMODE_WRITE) {
5024		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
5025
5026		if (filter_hash) {
5027			orig_hash = &iter->ops->func_hash->filter_hash;
5028			if (iter->tr && !list_empty(&iter->tr->mod_trace))
5029				iter->hash->flags |= FTRACE_HASH_FL_MOD;
5030		} else
5031			orig_hash = &iter->ops->func_hash->notrace_hash;
5032
5033		mutex_lock(&ftrace_lock);
5034		ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
5035						      iter->hash, filter_hash);
5036		mutex_unlock(&ftrace_lock);
5037	} else {
5038		/* For read only, the hash is the ops hash */
5039		iter->hash = NULL;
5040	}
5041
5042	mutex_unlock(&iter->ops->func_hash->regex_lock);
5043	free_ftrace_hash(iter->hash);
 
 
5044	kfree(iter);
5045
5046	return 0;
5047}
5048
5049static const struct file_operations ftrace_avail_fops = {
5050	.open = ftrace_avail_open,
5051	.read = seq_read,
5052	.llseek = seq_lseek,
5053	.release = seq_release_private,
5054};
5055
5056static const struct file_operations ftrace_enabled_fops = {
5057	.open = ftrace_enabled_open,
5058	.read = seq_read,
5059	.llseek = seq_lseek,
5060	.release = seq_release_private,
5061};
5062
5063static const struct file_operations ftrace_filter_fops = {
5064	.open = ftrace_filter_open,
5065	.read = seq_read,
5066	.write = ftrace_filter_write,
5067	.llseek = tracing_lseek,
5068	.release = ftrace_regex_release,
5069};
5070
5071static const struct file_operations ftrace_notrace_fops = {
5072	.open = ftrace_notrace_open,
5073	.read = seq_read,
5074	.write = ftrace_notrace_write,
5075	.llseek = tracing_lseek,
5076	.release = ftrace_regex_release,
5077};
5078
5079#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5080
5081static DEFINE_MUTEX(graph_lock);
5082
5083struct ftrace_hash *ftrace_graph_hash = EMPTY_HASH;
5084struct ftrace_hash *ftrace_graph_notrace_hash = EMPTY_HASH;
5085
5086enum graph_filter_type {
5087	GRAPH_FILTER_NOTRACE	= 0,
5088	GRAPH_FILTER_FUNCTION,
5089};
5090
5091#define FTRACE_GRAPH_EMPTY	((void *)1)
5092
5093struct ftrace_graph_data {
5094	struct ftrace_hash		*hash;
5095	struct ftrace_func_entry	*entry;
5096	int				idx;   /* for hash table iteration */
5097	enum graph_filter_type		type;
5098	struct ftrace_hash		*new_hash;
5099	const struct seq_operations	*seq_ops;
5100	struct trace_parser		parser;
5101};
5102
5103static void *
5104__g_next(struct seq_file *m, loff_t *pos)
5105{
5106	struct ftrace_graph_data *fgd = m->private;
5107	struct ftrace_func_entry *entry = fgd->entry;
5108	struct hlist_head *head;
5109	int i, idx = fgd->idx;
5110
5111	if (*pos >= fgd->hash->count)
5112		return NULL;
5113
5114	if (entry) {
5115		hlist_for_each_entry_continue(entry, hlist) {
5116			fgd->entry = entry;
5117			return entry;
5118		}
5119
5120		idx++;
5121	}
5122
5123	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
5124		head = &fgd->hash->buckets[i];
5125		hlist_for_each_entry(entry, head, hlist) {
5126			fgd->entry = entry;
5127			fgd->idx = i;
5128			return entry;
5129		}
5130	}
5131	return NULL;
5132}
5133
5134static void *
5135g_next(struct seq_file *m, void *v, loff_t *pos)
5136{
5137	(*pos)++;
5138	return __g_next(m, pos);
5139}
5140
5141static void *g_start(struct seq_file *m, loff_t *pos)
5142{
5143	struct ftrace_graph_data *fgd = m->private;
5144
5145	mutex_lock(&graph_lock);
5146
5147	if (fgd->type == GRAPH_FILTER_FUNCTION)
5148		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
5149					lockdep_is_held(&graph_lock));
5150	else
5151		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
5152					lockdep_is_held(&graph_lock));
5153
5154	/* Nothing, tell g_show to print all functions are enabled */
5155	if (ftrace_hash_empty(fgd->hash) && !*pos)
5156		return FTRACE_GRAPH_EMPTY;
5157
5158	fgd->idx = 0;
5159	fgd->entry = NULL;
5160	return __g_next(m, pos);
5161}
5162
5163static void g_stop(struct seq_file *m, void *p)
5164{
5165	mutex_unlock(&graph_lock);
5166}
5167
5168static int g_show(struct seq_file *m, void *v)
5169{
5170	struct ftrace_func_entry *entry = v;
5171
5172	if (!entry)
5173		return 0;
5174
5175	if (entry == FTRACE_GRAPH_EMPTY) {
5176		struct ftrace_graph_data *fgd = m->private;
5177
5178		if (fgd->type == GRAPH_FILTER_FUNCTION)
5179			seq_puts(m, "#### all functions enabled ####\n");
5180		else
5181			seq_puts(m, "#### no functions disabled ####\n");
5182		return 0;
5183	}
5184
5185	seq_printf(m, "%ps\n", (void *)entry->ip);
5186
5187	return 0;
5188}
5189
5190static const struct seq_operations ftrace_graph_seq_ops = {
5191	.start = g_start,
5192	.next = g_next,
5193	.stop = g_stop,
5194	.show = g_show,
5195};
5196
5197static int
5198__ftrace_graph_open(struct inode *inode, struct file *file,
5199		    struct ftrace_graph_data *fgd)
5200{
5201	int ret = 0;
5202	struct ftrace_hash *new_hash = NULL;
5203
 
 
 
 
5204	if (file->f_mode & FMODE_WRITE) {
5205		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
5206
5207		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
5208			return -ENOMEM;
5209
5210		if (file->f_flags & O_TRUNC)
5211			new_hash = alloc_ftrace_hash(size_bits);
5212		else
5213			new_hash = alloc_and_copy_ftrace_hash(size_bits,
5214							      fgd->hash);
5215		if (!new_hash) {
5216			ret = -ENOMEM;
5217			goto out;
5218		}
5219	}
5220
5221	if (file->f_mode & FMODE_READ) {
5222		ret = seq_open(file, &ftrace_graph_seq_ops);
5223		if (!ret) {
5224			struct seq_file *m = file->private_data;
5225			m->private = fgd;
5226		} else {
5227			/* Failed */
5228			free_ftrace_hash(new_hash);
5229			new_hash = NULL;
5230		}
5231	} else
5232		file->private_data = fgd;
5233
5234out:
5235	if (ret < 0 && file->f_mode & FMODE_WRITE)
5236		trace_parser_put(&fgd->parser);
5237
5238	fgd->new_hash = new_hash;
5239
5240	/*
5241	 * All uses of fgd->hash must be taken with the graph_lock
5242	 * held. The graph_lock is going to be released, so force
5243	 * fgd->hash to be reinitialized when it is taken again.
5244	 */
5245	fgd->hash = NULL;
5246
5247	return ret;
5248}
5249
5250static int
5251ftrace_graph_open(struct inode *inode, struct file *file)
5252{
5253	struct ftrace_graph_data *fgd;
5254	int ret;
5255
5256	if (unlikely(ftrace_disabled))
5257		return -ENODEV;
5258
5259	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
5260	if (fgd == NULL)
5261		return -ENOMEM;
5262
5263	mutex_lock(&graph_lock);
5264
5265	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
5266					lockdep_is_held(&graph_lock));
5267	fgd->type = GRAPH_FILTER_FUNCTION;
5268	fgd->seq_ops = &ftrace_graph_seq_ops;
5269
5270	ret = __ftrace_graph_open(inode, file, fgd);
5271	if (ret < 0)
5272		kfree(fgd);
5273
5274	mutex_unlock(&graph_lock);
5275	return ret;
5276}
5277
5278static int
5279ftrace_graph_notrace_open(struct inode *inode, struct file *file)
5280{
5281	struct ftrace_graph_data *fgd;
5282	int ret;
5283
5284	if (unlikely(ftrace_disabled))
5285		return -ENODEV;
5286
5287	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
5288	if (fgd == NULL)
5289		return -ENOMEM;
5290
5291	mutex_lock(&graph_lock);
5292
5293	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
5294					lockdep_is_held(&graph_lock));
5295	fgd->type = GRAPH_FILTER_NOTRACE;
5296	fgd->seq_ops = &ftrace_graph_seq_ops;
5297
5298	ret = __ftrace_graph_open(inode, file, fgd);
5299	if (ret < 0)
5300		kfree(fgd);
5301
5302	mutex_unlock(&graph_lock);
5303	return ret;
5304}
5305
5306static int
5307ftrace_graph_release(struct inode *inode, struct file *file)
5308{
5309	struct ftrace_graph_data *fgd;
5310	struct ftrace_hash *old_hash, *new_hash;
5311	struct trace_parser *parser;
5312	int ret = 0;
5313
5314	if (file->f_mode & FMODE_READ) {
5315		struct seq_file *m = file->private_data;
5316
5317		fgd = m->private;
5318		seq_release(inode, file);
5319	} else {
5320		fgd = file->private_data;
5321	}
5322
5323
5324	if (file->f_mode & FMODE_WRITE) {
5325
5326		parser = &fgd->parser;
5327
5328		if (trace_parser_loaded((parser))) {
5329			ret = ftrace_graph_set_hash(fgd->new_hash,
5330						    parser->buffer);
5331		}
5332
5333		trace_parser_put(parser);
5334
5335		new_hash = __ftrace_hash_move(fgd->new_hash);
5336		if (!new_hash) {
5337			ret = -ENOMEM;
5338			goto out;
5339		}
5340
5341		mutex_lock(&graph_lock);
5342
5343		if (fgd->type == GRAPH_FILTER_FUNCTION) {
5344			old_hash = rcu_dereference_protected(ftrace_graph_hash,
5345					lockdep_is_held(&graph_lock));
5346			rcu_assign_pointer(ftrace_graph_hash, new_hash);
5347		} else {
5348			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
5349					lockdep_is_held(&graph_lock));
5350			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
5351		}
5352
5353		mutex_unlock(&graph_lock);
5354
5355		/* Wait till all users are no longer using the old hash */
5356		synchronize_sched();
 
 
 
 
 
 
 
5357
5358		free_ftrace_hash(old_hash);
5359	}
5360
5361 out:
5362	free_ftrace_hash(fgd->new_hash);
5363	kfree(fgd);
5364
5365	return ret;
5366}
5367
5368static int
5369ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
5370{
5371	struct ftrace_glob func_g;
5372	struct dyn_ftrace *rec;
5373	struct ftrace_page *pg;
5374	struct ftrace_func_entry *entry;
5375	int fail = 1;
5376	int not;
5377
5378	/* decode regex */
5379	func_g.type = filter_parse_regex(buffer, strlen(buffer),
5380					 &func_g.search, &not);
5381
5382	func_g.len = strlen(func_g.search);
5383
5384	mutex_lock(&ftrace_lock);
5385
5386	if (unlikely(ftrace_disabled)) {
5387		mutex_unlock(&ftrace_lock);
5388		return -ENODEV;
5389	}
5390
5391	do_for_each_ftrace_rec(pg, rec) {
5392
5393		if (rec->flags & FTRACE_FL_DISABLED)
5394			continue;
5395
5396		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
5397			entry = ftrace_lookup_ip(hash, rec->ip);
5398
5399			if (!not) {
5400				fail = 0;
5401
5402				if (entry)
5403					continue;
5404				if (add_hash_entry(hash, rec->ip) < 0)
5405					goto out;
5406			} else {
5407				if (entry) {
5408					free_hash_entry(hash, entry);
5409					fail = 0;
5410				}
5411			}
5412		}
5413	} while_for_each_ftrace_rec();
5414out:
5415	mutex_unlock(&ftrace_lock);
5416
5417	if (fail)
5418		return -EINVAL;
5419
5420	return 0;
5421}
5422
5423static ssize_t
5424ftrace_graph_write(struct file *file, const char __user *ubuf,
5425		   size_t cnt, loff_t *ppos)
5426{
5427	ssize_t read, ret = 0;
5428	struct ftrace_graph_data *fgd = file->private_data;
5429	struct trace_parser *parser;
5430
5431	if (!cnt)
5432		return 0;
5433
5434	/* Read mode uses seq functions */
5435	if (file->f_mode & FMODE_READ) {
5436		struct seq_file *m = file->private_data;
5437		fgd = m->private;
5438	}
5439
5440	parser = &fgd->parser;
5441
5442	read = trace_get_user(parser, ubuf, cnt, ppos);
5443
5444	if (read >= 0 && trace_parser_loaded(parser) &&
5445	    !trace_parser_cont(parser)) {
5446
5447		ret = ftrace_graph_set_hash(fgd->new_hash,
5448					    parser->buffer);
5449		trace_parser_clear(parser);
5450	}
5451
5452	if (!ret)
5453		ret = read;
5454
5455	return ret;
5456}
5457
5458static const struct file_operations ftrace_graph_fops = {
5459	.open		= ftrace_graph_open,
5460	.read		= seq_read,
5461	.write		= ftrace_graph_write,
5462	.llseek		= tracing_lseek,
5463	.release	= ftrace_graph_release,
5464};
5465
5466static const struct file_operations ftrace_graph_notrace_fops = {
5467	.open		= ftrace_graph_notrace_open,
5468	.read		= seq_read,
5469	.write		= ftrace_graph_write,
5470	.llseek		= tracing_lseek,
5471	.release	= ftrace_graph_release,
5472};
5473#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
5474
5475void ftrace_create_filter_files(struct ftrace_ops *ops,
5476				struct dentry *parent)
5477{
5478
5479	trace_create_file("set_ftrace_filter", 0644, parent,
5480			  ops, &ftrace_filter_fops);
5481
5482	trace_create_file("set_ftrace_notrace", 0644, parent,
5483			  ops, &ftrace_notrace_fops);
5484}
5485
5486/*
5487 * The name "destroy_filter_files" is really a misnomer. Although
5488 * in the future, it may actualy delete the files, but this is
5489 * really intended to make sure the ops passed in are disabled
5490 * and that when this function returns, the caller is free to
5491 * free the ops.
5492 *
5493 * The "destroy" name is only to match the "create" name that this
5494 * should be paired with.
5495 */
5496void ftrace_destroy_filter_files(struct ftrace_ops *ops)
5497{
5498	mutex_lock(&ftrace_lock);
5499	if (ops->flags & FTRACE_OPS_FL_ENABLED)
5500		ftrace_shutdown(ops, 0);
5501	ops->flags |= FTRACE_OPS_FL_DELETED;
 
5502	mutex_unlock(&ftrace_lock);
5503}
5504
5505static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
5506{
5507
5508	trace_create_file("available_filter_functions", 0444,
5509			d_tracer, NULL, &ftrace_avail_fops);
5510
5511	trace_create_file("enabled_functions", 0444,
5512			d_tracer, NULL, &ftrace_enabled_fops);
5513
5514	ftrace_create_filter_files(&global_ops, d_tracer);
5515
5516#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5517	trace_create_file("set_graph_function", 0644, d_tracer,
5518				    NULL,
5519				    &ftrace_graph_fops);
5520	trace_create_file("set_graph_notrace", 0644, d_tracer,
5521				    NULL,
5522				    &ftrace_graph_notrace_fops);
5523#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
5524
5525	return 0;
5526}
5527
5528static int ftrace_cmp_ips(const void *a, const void *b)
5529{
5530	const unsigned long *ipa = a;
5531	const unsigned long *ipb = b;
5532
5533	if (*ipa > *ipb)
5534		return 1;
5535	if (*ipa < *ipb)
5536		return -1;
5537	return 0;
5538}
5539
5540static int ftrace_process_locs(struct module *mod,
5541			       unsigned long *start,
5542			       unsigned long *end)
5543{
5544	struct ftrace_page *start_pg;
5545	struct ftrace_page *pg;
5546	struct dyn_ftrace *rec;
5547	unsigned long count;
5548	unsigned long *p;
5549	unsigned long addr;
5550	unsigned long flags = 0; /* Shut up gcc */
5551	int ret = -ENOMEM;
5552
5553	count = end - start;
5554
5555	if (!count)
5556		return 0;
5557
5558	sort(start, count, sizeof(*start),
5559	     ftrace_cmp_ips, NULL);
5560
5561	start_pg = ftrace_allocate_pages(count);
5562	if (!start_pg)
5563		return -ENOMEM;
5564
5565	mutex_lock(&ftrace_lock);
5566
5567	/*
5568	 * Core and each module needs their own pages, as
5569	 * modules will free them when they are removed.
5570	 * Force a new page to be allocated for modules.
5571	 */
5572	if (!mod) {
5573		WARN_ON(ftrace_pages || ftrace_pages_start);
5574		/* First initialization */
5575		ftrace_pages = ftrace_pages_start = start_pg;
5576	} else {
5577		if (!ftrace_pages)
5578			goto out;
5579
5580		if (WARN_ON(ftrace_pages->next)) {
5581			/* Hmm, we have free pages? */
5582			while (ftrace_pages->next)
5583				ftrace_pages = ftrace_pages->next;
5584		}
5585
5586		ftrace_pages->next = start_pg;
5587	}
5588
5589	p = start;
5590	pg = start_pg;
5591	while (p < end) {
5592		addr = ftrace_call_adjust(*p++);
5593		/*
5594		 * Some architecture linkers will pad between
5595		 * the different mcount_loc sections of different
5596		 * object files to satisfy alignments.
5597		 * Skip any NULL pointers.
5598		 */
5599		if (!addr)
5600			continue;
5601
5602		if (pg->index == pg->size) {
5603			/* We should have allocated enough */
5604			if (WARN_ON(!pg->next))
5605				break;
5606			pg = pg->next;
5607		}
5608
5609		rec = &pg->records[pg->index++];
5610		rec->ip = addr;
5611	}
5612
5613	/* We should have used all pages */
5614	WARN_ON(pg->next);
5615
5616	/* Assign the last page to ftrace_pages */
5617	ftrace_pages = pg;
5618
5619	/*
5620	 * We only need to disable interrupts on start up
5621	 * because we are modifying code that an interrupt
5622	 * may execute, and the modification is not atomic.
5623	 * But for modules, nothing runs the code we modify
5624	 * until we are finished with it, and there's no
5625	 * reason to cause large interrupt latencies while we do it.
5626	 */
5627	if (!mod)
5628		local_irq_save(flags);
5629	ftrace_update_code(mod, start_pg);
5630	if (!mod)
5631		local_irq_restore(flags);
5632	ret = 0;
5633 out:
5634	mutex_unlock(&ftrace_lock);
5635
5636	return ret;
5637}
5638
5639struct ftrace_mod_func {
5640	struct list_head	list;
5641	char			*name;
5642	unsigned long		ip;
5643	unsigned int		size;
5644};
5645
5646struct ftrace_mod_map {
5647	struct rcu_head		rcu;
5648	struct list_head	list;
5649	struct module		*mod;
5650	unsigned long		start_addr;
5651	unsigned long		end_addr;
5652	struct list_head	funcs;
5653	unsigned int		num_funcs;
5654};
5655
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5656#ifdef CONFIG_MODULES
5657
5658#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
5659
5660static LIST_HEAD(ftrace_mod_maps);
5661
5662static int referenced_filters(struct dyn_ftrace *rec)
5663{
5664	struct ftrace_ops *ops;
5665	int cnt = 0;
5666
5667	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
5668		if (ops_references_rec(ops, rec))
5669		    cnt++;
 
 
 
 
 
 
 
 
 
 
 
5670	}
5671
5672	return cnt;
5673}
5674
5675static void
5676clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
5677{
5678	struct ftrace_func_entry *entry;
5679	struct dyn_ftrace *rec;
5680	int i;
5681
5682	if (ftrace_hash_empty(hash))
5683		return;
5684
5685	for (i = 0; i < pg->index; i++) {
5686		rec = &pg->records[i];
5687		entry = __ftrace_lookup_ip(hash, rec->ip);
5688		/*
5689		 * Do not allow this rec to match again.
5690		 * Yeah, it may waste some memory, but will be removed
5691		 * if/when the hash is modified again.
5692		 */
5693		if (entry)
5694			entry->ip = 0;
5695	}
5696}
5697
5698/* Clear any records from hashs */
5699static void clear_mod_from_hashes(struct ftrace_page *pg)
5700{
5701	struct trace_array *tr;
5702
5703	mutex_lock(&trace_types_lock);
5704	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5705		if (!tr->ops || !tr->ops->func_hash)
5706			continue;
5707		mutex_lock(&tr->ops->func_hash->regex_lock);
5708		clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
5709		clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
5710		mutex_unlock(&tr->ops->func_hash->regex_lock);
5711	}
5712	mutex_unlock(&trace_types_lock);
5713}
5714
5715static void ftrace_free_mod_map(struct rcu_head *rcu)
5716{
5717	struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
5718	struct ftrace_mod_func *mod_func;
5719	struct ftrace_mod_func *n;
5720
5721	/* All the contents of mod_map are now not visible to readers */
5722	list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
5723		kfree(mod_func->name);
5724		list_del(&mod_func->list);
5725		kfree(mod_func);
5726	}
5727
5728	kfree(mod_map);
5729}
5730
5731void ftrace_release_mod(struct module *mod)
5732{
5733	struct ftrace_mod_map *mod_map;
5734	struct ftrace_mod_map *n;
5735	struct dyn_ftrace *rec;
5736	struct ftrace_page **last_pg;
5737	struct ftrace_page *tmp_page = NULL;
5738	struct ftrace_page *pg;
5739	int order;
5740
5741	mutex_lock(&ftrace_lock);
5742
5743	if (ftrace_disabled)
5744		goto out_unlock;
5745
5746	list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
5747		if (mod_map->mod == mod) {
5748			list_del_rcu(&mod_map->list);
5749			call_rcu_sched(&mod_map->rcu, ftrace_free_mod_map);
5750			break;
5751		}
5752	}
5753
5754	/*
5755	 * Each module has its own ftrace_pages, remove
5756	 * them from the list.
5757	 */
5758	last_pg = &ftrace_pages_start;
5759	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
5760		rec = &pg->records[0];
5761		if (within_module_core(rec->ip, mod) ||
5762		    within_module_init(rec->ip, mod)) {
5763			/*
5764			 * As core pages are first, the first
5765			 * page should never be a module page.
5766			 */
5767			if (WARN_ON(pg == ftrace_pages_start))
5768				goto out_unlock;
5769
5770			/* Check if we are deleting the last page */
5771			if (pg == ftrace_pages)
5772				ftrace_pages = next_to_ftrace_page(last_pg);
5773
5774			ftrace_update_tot_cnt -= pg->index;
5775			*last_pg = pg->next;
5776
5777			pg->next = tmp_page;
5778			tmp_page = pg;
5779		} else
5780			last_pg = &pg->next;
5781	}
5782 out_unlock:
5783	mutex_unlock(&ftrace_lock);
5784
5785	for (pg = tmp_page; pg; pg = tmp_page) {
5786
5787		/* Needs to be called outside of ftrace_lock */
5788		clear_mod_from_hashes(pg);
5789
5790		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
5791		free_pages((unsigned long)pg->records, order);
5792		tmp_page = pg->next;
5793		kfree(pg);
 
 
5794	}
5795}
5796
5797void ftrace_module_enable(struct module *mod)
5798{
5799	struct dyn_ftrace *rec;
5800	struct ftrace_page *pg;
5801
5802	mutex_lock(&ftrace_lock);
5803
5804	if (ftrace_disabled)
5805		goto out_unlock;
5806
5807	/*
5808	 * If the tracing is enabled, go ahead and enable the record.
5809	 *
5810	 * The reason not to enable the record immediatelly is the
5811	 * inherent check of ftrace_make_nop/ftrace_make_call for
5812	 * correct previous instructions.  Making first the NOP
5813	 * conversion puts the module to the correct state, thus
5814	 * passing the ftrace_make_call check.
5815	 *
5816	 * We also delay this to after the module code already set the
5817	 * text to read-only, as we now need to set it back to read-write
5818	 * so that we can modify the text.
5819	 */
5820	if (ftrace_start_up)
5821		ftrace_arch_code_modify_prepare();
5822
5823	do_for_each_ftrace_rec(pg, rec) {
5824		int cnt;
5825		/*
5826		 * do_for_each_ftrace_rec() is a double loop.
5827		 * module text shares the pg. If a record is
5828		 * not part of this module, then skip this pg,
5829		 * which the "break" will do.
5830		 */
5831		if (!within_module_core(rec->ip, mod) &&
5832		    !within_module_init(rec->ip, mod))
5833			break;
5834
5835		cnt = 0;
5836
5837		/*
5838		 * When adding a module, we need to check if tracers are
5839		 * currently enabled and if they are, and can trace this record,
5840		 * we need to enable the module functions as well as update the
5841		 * reference counts for those function records.
5842		 */
5843		if (ftrace_start_up)
5844			cnt += referenced_filters(rec);
5845
5846		/* This clears FTRACE_FL_DISABLED */
5847		rec->flags = cnt;
5848
5849		if (ftrace_start_up && cnt) {
5850			int failed = __ftrace_replace_code(rec, 1);
5851			if (failed) {
5852				ftrace_bug(failed, rec);
5853				goto out_loop;
5854			}
5855		}
5856
5857	} while_for_each_ftrace_rec();
5858
5859 out_loop:
5860	if (ftrace_start_up)
5861		ftrace_arch_code_modify_post_process();
5862
5863 out_unlock:
5864	mutex_unlock(&ftrace_lock);
5865
5866	process_cached_mods(mod->name);
5867}
5868
5869void ftrace_module_init(struct module *mod)
5870{
5871	if (ftrace_disabled || !mod->num_ftrace_callsites)
5872		return;
5873
5874	ftrace_process_locs(mod, mod->ftrace_callsites,
5875			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5876}
5877
5878static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
5879				struct dyn_ftrace *rec)
5880{
5881	struct ftrace_mod_func *mod_func;
5882	unsigned long symsize;
5883	unsigned long offset;
5884	char str[KSYM_SYMBOL_LEN];
5885	char *modname;
5886	const char *ret;
5887
5888	ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
5889	if (!ret)
5890		return;
5891
5892	mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL);
5893	if (!mod_func)
5894		return;
5895
5896	mod_func->name = kstrdup(str, GFP_KERNEL);
5897	if (!mod_func->name) {
5898		kfree(mod_func);
5899		return;
5900	}
5901
5902	mod_func->ip = rec->ip - offset;
5903	mod_func->size = symsize;
5904
5905	mod_map->num_funcs++;
5906
5907	list_add_rcu(&mod_func->list, &mod_map->funcs);
5908}
5909
5910static struct ftrace_mod_map *
5911allocate_ftrace_mod_map(struct module *mod,
5912			unsigned long start, unsigned long end)
5913{
5914	struct ftrace_mod_map *mod_map;
5915
5916	mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL);
5917	if (!mod_map)
5918		return NULL;
5919
5920	mod_map->mod = mod;
5921	mod_map->start_addr = start;
5922	mod_map->end_addr = end;
5923	mod_map->num_funcs = 0;
5924
5925	INIT_LIST_HEAD_RCU(&mod_map->funcs);
5926
5927	list_add_rcu(&mod_map->list, &ftrace_mod_maps);
5928
5929	return mod_map;
5930}
5931
5932static const char *
5933ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
5934			   unsigned long addr, unsigned long *size,
5935			   unsigned long *off, char *sym)
5936{
5937	struct ftrace_mod_func *found_func =  NULL;
5938	struct ftrace_mod_func *mod_func;
5939
5940	list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
5941		if (addr >= mod_func->ip &&
5942		    addr < mod_func->ip + mod_func->size) {
5943			found_func = mod_func;
5944			break;
5945		}
5946	}
5947
5948	if (found_func) {
5949		if (size)
5950			*size = found_func->size;
5951		if (off)
5952			*off = addr - found_func->ip;
5953		if (sym)
5954			strlcpy(sym, found_func->name, KSYM_NAME_LEN);
5955
5956		return found_func->name;
5957	}
5958
5959	return NULL;
5960}
5961
5962const char *
5963ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
5964		   unsigned long *off, char **modname, char *sym)
5965{
5966	struct ftrace_mod_map *mod_map;
5967	const char *ret = NULL;
5968
5969	/* mod_map is freed via call_rcu_sched() */
5970	preempt_disable();
5971	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
5972		ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
5973		if (ret) {
5974			if (modname)
5975				*modname = mod_map->mod->name;
5976			break;
5977		}
5978	}
5979	preempt_enable();
5980
5981	return ret;
5982}
5983
5984int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
5985			   char *type, char *name,
5986			   char *module_name, int *exported)
5987{
5988	struct ftrace_mod_map *mod_map;
5989	struct ftrace_mod_func *mod_func;
 
5990
5991	preempt_disable();
5992	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
5993
5994		if (symnum >= mod_map->num_funcs) {
5995			symnum -= mod_map->num_funcs;
5996			continue;
5997		}
5998
5999		list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
6000			if (symnum > 1) {
6001				symnum--;
6002				continue;
6003			}
6004
6005			*value = mod_func->ip;
6006			*type = 'T';
6007			strlcpy(name, mod_func->name, KSYM_NAME_LEN);
6008			strlcpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
6009			*exported = 1;
6010			preempt_enable();
6011			return 0;
6012		}
6013		WARN_ON(1);
6014		break;
6015	}
 
 
6016	preempt_enable();
6017	return -ERANGE;
6018}
6019
6020#else
6021static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
6022				struct dyn_ftrace *rec) { }
6023static inline struct ftrace_mod_map *
6024allocate_ftrace_mod_map(struct module *mod,
6025			unsigned long start, unsigned long end)
6026{
6027	return NULL;
6028}
 
 
 
 
 
 
 
 
 
 
 
 
6029#endif /* CONFIG_MODULES */
6030
6031struct ftrace_init_func {
6032	struct list_head list;
6033	unsigned long ip;
6034};
6035
6036/* Clear any init ips from hashes */
6037static void
6038clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
6039{
6040	struct ftrace_func_entry *entry;
6041
6042	if (ftrace_hash_empty(hash))
6043		return;
6044
6045	entry = __ftrace_lookup_ip(hash, func->ip);
6046
6047	/*
6048	 * Do not allow this rec to match again.
6049	 * Yeah, it may waste some memory, but will be removed
6050	 * if/when the hash is modified again.
6051	 */
6052	if (entry)
6053		entry->ip = 0;
6054}
6055
6056static void
6057clear_func_from_hashes(struct ftrace_init_func *func)
6058{
6059	struct trace_array *tr;
6060
6061	mutex_lock(&trace_types_lock);
6062	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6063		if (!tr->ops || !tr->ops->func_hash)
6064			continue;
6065		mutex_lock(&tr->ops->func_hash->regex_lock);
6066		clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
6067		clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
6068		mutex_unlock(&tr->ops->func_hash->regex_lock);
6069	}
6070	mutex_unlock(&trace_types_lock);
6071}
6072
6073static void add_to_clear_hash_list(struct list_head *clear_list,
6074				   struct dyn_ftrace *rec)
6075{
6076	struct ftrace_init_func *func;
6077
6078	func = kmalloc(sizeof(*func), GFP_KERNEL);
6079	if (!func) {
6080		WARN_ONCE(1, "alloc failure, ftrace filter could be stale\n");
6081		return;
6082	}
6083
6084	func->ip = rec->ip;
6085	list_add(&func->list, clear_list);
6086}
6087
6088void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
6089{
6090	unsigned long start = (unsigned long)(start_ptr);
6091	unsigned long end = (unsigned long)(end_ptr);
6092	struct ftrace_page **last_pg = &ftrace_pages_start;
6093	struct ftrace_page *pg;
6094	struct dyn_ftrace *rec;
6095	struct dyn_ftrace key;
6096	struct ftrace_mod_map *mod_map = NULL;
6097	struct ftrace_init_func *func, *func_next;
6098	struct list_head clear_hash;
6099	int order;
6100
6101	INIT_LIST_HEAD(&clear_hash);
6102
6103	key.ip = start;
6104	key.flags = end;	/* overload flags, as it is unsigned long */
6105
6106	mutex_lock(&ftrace_lock);
6107
6108	/*
6109	 * If we are freeing module init memory, then check if
6110	 * any tracer is active. If so, we need to save a mapping of
6111	 * the module functions being freed with the address.
6112	 */
6113	if (mod && ftrace_ops_list != &ftrace_list_end)
6114		mod_map = allocate_ftrace_mod_map(mod, start, end);
6115
6116	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
6117		if (end < pg->records[0].ip ||
6118		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
6119			continue;
6120 again:
6121		rec = bsearch(&key, pg->records, pg->index,
6122			      sizeof(struct dyn_ftrace),
6123			      ftrace_cmp_recs);
6124		if (!rec)
6125			continue;
6126
6127		/* rec will be cleared from hashes after ftrace_lock unlock */
6128		add_to_clear_hash_list(&clear_hash, rec);
6129
6130		if (mod_map)
6131			save_ftrace_mod_rec(mod_map, rec);
6132
6133		pg->index--;
6134		ftrace_update_tot_cnt--;
6135		if (!pg->index) {
6136			*last_pg = pg->next;
6137			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
6138			free_pages((unsigned long)pg->records, order);
 
 
6139			kfree(pg);
6140			pg = container_of(last_pg, struct ftrace_page, next);
6141			if (!(*last_pg))
6142				ftrace_pages = pg;
6143			continue;
6144		}
6145		memmove(rec, rec + 1,
6146			(pg->index - (rec - pg->records)) * sizeof(*rec));
6147		/* More than one function may be in this block */
6148		goto again;
6149	}
6150	mutex_unlock(&ftrace_lock);
6151
6152	list_for_each_entry_safe(func, func_next, &clear_hash, list) {
6153		clear_func_from_hashes(func);
6154		kfree(func);
6155	}
6156}
6157
6158void __init ftrace_free_init_mem(void)
6159{
6160	void *start = (void *)(&__init_begin);
6161	void *end = (void *)(&__init_end);
6162
6163	ftrace_free_mem(NULL, start, end);
6164}
6165
6166void __init ftrace_init(void)
6167{
6168	extern unsigned long __start_mcount_loc[];
6169	extern unsigned long __stop_mcount_loc[];
6170	unsigned long count, flags;
6171	int ret;
6172
6173	local_irq_save(flags);
6174	ret = ftrace_dyn_arch_init();
6175	local_irq_restore(flags);
6176	if (ret)
6177		goto failed;
6178
6179	count = __stop_mcount_loc - __start_mcount_loc;
6180	if (!count) {
6181		pr_info("ftrace: No functions to be traced?\n");
6182		goto failed;
6183	}
6184
6185	pr_info("ftrace: allocating %ld entries in %ld pages\n",
6186		count, count / ENTRIES_PER_PAGE + 1);
6187
6188	last_ftrace_enabled = ftrace_enabled = 1;
6189
6190	ret = ftrace_process_locs(NULL,
6191				  __start_mcount_loc,
6192				  __stop_mcount_loc);
6193
 
 
 
6194	set_ftrace_early_filters();
6195
6196	return;
6197 failed:
6198	ftrace_disabled = 1;
6199}
6200
6201/* Do nothing if arch does not support this */
6202void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
6203{
6204}
6205
6206static void ftrace_update_trampoline(struct ftrace_ops *ops)
6207{
 
 
6208	arch_ftrace_update_trampoline(ops);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6209}
6210
6211void ftrace_init_trace_array(struct trace_array *tr)
6212{
6213	INIT_LIST_HEAD(&tr->func_probes);
6214	INIT_LIST_HEAD(&tr->mod_trace);
6215	INIT_LIST_HEAD(&tr->mod_notrace);
6216}
6217#else
6218
6219static struct ftrace_ops global_ops = {
6220	.func			= ftrace_stub,
6221	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
6222				  FTRACE_OPS_FL_INITIALIZED |
6223				  FTRACE_OPS_FL_PID,
6224};
6225
6226static int __init ftrace_nodyn_init(void)
6227{
6228	ftrace_enabled = 1;
6229	return 0;
6230}
6231core_initcall(ftrace_nodyn_init);
6232
6233static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
6234static inline void ftrace_startup_enable(int command) { }
6235static inline void ftrace_startup_all(int command) { }
6236/* Keep as macros so we do not need to define the commands */
6237# define ftrace_startup(ops, command)					\
6238	({								\
6239		int ___ret = __register_ftrace_function(ops);		\
6240		if (!___ret)						\
6241			(ops)->flags |= FTRACE_OPS_FL_ENABLED;		\
6242		___ret;							\
6243	})
6244# define ftrace_shutdown(ops, command)					\
6245	({								\
6246		int ___ret = __unregister_ftrace_function(ops);		\
6247		if (!___ret)						\
6248			(ops)->flags &= ~FTRACE_OPS_FL_ENABLED;		\
6249		___ret;							\
6250	})
6251
6252# define ftrace_startup_sysctl()	do { } while (0)
6253# define ftrace_shutdown_sysctl()	do { } while (0)
6254
6255static inline int
6256ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
6257{
6258	return 1;
6259}
6260
6261static void ftrace_update_trampoline(struct ftrace_ops *ops)
6262{
6263}
6264
6265#endif /* CONFIG_DYNAMIC_FTRACE */
6266
6267__init void ftrace_init_global_array_ops(struct trace_array *tr)
6268{
6269	tr->ops = &global_ops;
6270	tr->ops->private = tr;
6271	ftrace_init_trace_array(tr);
6272}
6273
6274void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
6275{
6276	/* If we filter on pids, update to use the pid function */
6277	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
6278		if (WARN_ON(tr->ops->func != ftrace_stub))
6279			printk("ftrace ops had %pS for function\n",
6280			       tr->ops->func);
6281	}
6282	tr->ops->func = func;
6283	tr->ops->private = tr;
6284}
6285
6286void ftrace_reset_array_ops(struct trace_array *tr)
6287{
6288	tr->ops->func = ftrace_stub;
6289}
6290
6291static inline void
6292__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
6293		       struct ftrace_ops *ignored, struct pt_regs *regs)
6294{
6295	struct ftrace_ops *op;
6296	int bit;
6297
6298	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
6299	if (bit < 0)
6300		return;
6301
6302	/*
6303	 * Some of the ops may be dynamically allocated,
6304	 * they must be freed after a synchronize_sched().
6305	 */
6306	preempt_disable_notrace();
6307
6308	do_for_each_ftrace_op(op, ftrace_ops_list) {
 
 
 
6309		/*
6310		 * Check the following for each ops before calling their func:
6311		 *  if RCU flag is set, then rcu_is_watching() must be true
6312		 *  if PER_CPU is set, then ftrace_function_local_disable()
6313		 *                          must be false
6314		 *  Otherwise test if the ip matches the ops filter
6315		 *
6316		 * If any of the above fails then the op->func() is not executed.
6317		 */
6318		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
6319		    ftrace_ops_test(op, ip, regs)) {
6320			if (FTRACE_WARN_ON(!op->func)) {
6321				pr_warn("op=%p %pS\n", op, op);
6322				goto out;
6323			}
6324			op->func(ip, parent_ip, op, regs);
6325		}
6326	} while_for_each_ftrace_op(op);
6327out:
6328	preempt_enable_notrace();
6329	trace_clear_recursion(bit);
6330}
6331
6332/*
6333 * Some archs only support passing ip and parent_ip. Even though
6334 * the list function ignores the op parameter, we do not want any
6335 * C side effects, where a function is called without the caller
6336 * sending a third parameter.
6337 * Archs are to support both the regs and ftrace_ops at the same time.
6338 * If they support ftrace_ops, it is assumed they support regs.
6339 * If call backs want to use regs, they must either check for regs
6340 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
6341 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
6342 * An architecture can pass partial regs with ftrace_ops and still
6343 * set the ARCH_SUPPORTS_FTRACE_OPS.
6344 */
6345#if ARCH_SUPPORTS_FTRACE_OPS
6346static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
6347				 struct ftrace_ops *op, struct pt_regs *regs)
6348{
6349	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
6350}
 
6351#else
6352static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
6353{
6354	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
6355}
 
6356#endif
6357
6358/*
6359 * If there's only one function registered but it does not support
6360 * recursion, needs RCU protection and/or requires per cpu handling, then
6361 * this function will be called by the mcount trampoline.
6362 */
6363static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
6364				   struct ftrace_ops *op, struct pt_regs *regs)
6365{
6366	int bit;
6367
6368	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
6369		return;
6370
6371	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
6372	if (bit < 0)
6373		return;
6374
6375	preempt_disable_notrace();
6376
6377	op->func(ip, parent_ip, op, regs);
 
6378
6379	preempt_enable_notrace();
6380	trace_clear_recursion(bit);
6381}
 
6382
6383/**
6384 * ftrace_ops_get_func - get the function a trampoline should call
6385 * @ops: the ops to get the function for
6386 *
6387 * Normally the mcount trampoline will call the ops->func, but there
6388 * are times that it should not. For example, if the ops does not
6389 * have its own recursion protection, then it should call the
6390 * ftrace_ops_assist_func() instead.
6391 *
6392 * Returns the function that the trampoline should call for @ops.
6393 */
6394ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
6395{
6396	/*
6397	 * If the function does not handle recursion, needs to be RCU safe,
6398	 * or does per cpu logic, then we need to call the assist handler.
6399	 */
6400	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
6401	    ops->flags & FTRACE_OPS_FL_RCU)
6402		return ftrace_ops_assist_func;
6403
6404	return ops->func;
6405}
6406
6407static void
6408ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
6409		    struct task_struct *prev, struct task_struct *next)
6410{
6411	struct trace_array *tr = data;
6412	struct trace_pid_list *pid_list;
 
6413
6414	pid_list = rcu_dereference_sched(tr->function_pids);
 
6415
6416	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
6417		       trace_ignore_this_task(pid_list, next));
 
 
 
 
6418}
6419
6420static void
6421ftrace_pid_follow_sched_process_fork(void *data,
6422				     struct task_struct *self,
6423				     struct task_struct *task)
6424{
6425	struct trace_pid_list *pid_list;
6426	struct trace_array *tr = data;
6427
6428	pid_list = rcu_dereference_sched(tr->function_pids);
6429	trace_filter_add_remove_task(pid_list, self, task);
 
 
 
6430}
6431
6432static void
6433ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
6434{
6435	struct trace_pid_list *pid_list;
6436	struct trace_array *tr = data;
6437
6438	pid_list = rcu_dereference_sched(tr->function_pids);
6439	trace_filter_add_remove_task(pid_list, NULL, task);
 
 
 
6440}
6441
6442void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
6443{
6444	if (enable) {
6445		register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
6446						  tr);
6447		register_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
6448						  tr);
6449	} else {
6450		unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
6451						    tr);
6452		unregister_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
6453						    tr);
6454	}
6455}
6456
6457static void clear_ftrace_pids(struct trace_array *tr)
6458{
6459	struct trace_pid_list *pid_list;
 
6460	int cpu;
6461
6462	pid_list = rcu_dereference_protected(tr->function_pids,
6463					     lockdep_is_held(&ftrace_lock));
6464	if (!pid_list)
 
 
 
 
6465		return;
6466
6467	unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
 
 
 
 
 
6468
6469	for_each_possible_cpu(cpu)
6470		per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
6471
6472	rcu_assign_pointer(tr->function_pids, NULL);
 
6473
6474	/* Wait till all users are no longer using pid filtering */
6475	synchronize_sched();
 
 
 
6476
6477	trace_free_pid_list(pid_list);
 
6478}
6479
6480void ftrace_clear_pids(struct trace_array *tr)
6481{
6482	mutex_lock(&ftrace_lock);
6483
6484	clear_ftrace_pids(tr);
6485
6486	mutex_unlock(&ftrace_lock);
6487}
6488
6489static void ftrace_pid_reset(struct trace_array *tr)
6490{
6491	mutex_lock(&ftrace_lock);
6492	clear_ftrace_pids(tr);
6493
6494	ftrace_update_pid_func();
6495	ftrace_startup_all(0);
6496
6497	mutex_unlock(&ftrace_lock);
6498}
6499
6500/* Greater than any max PID */
6501#define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
6502
6503static void *fpid_start(struct seq_file *m, loff_t *pos)
6504	__acquires(RCU)
6505{
6506	struct trace_pid_list *pid_list;
6507	struct trace_array *tr = m->private;
6508
6509	mutex_lock(&ftrace_lock);
6510	rcu_read_lock_sched();
6511
6512	pid_list = rcu_dereference_sched(tr->function_pids);
6513
6514	if (!pid_list)
6515		return !(*pos) ? FTRACE_NO_PIDS : NULL;
6516
6517	return trace_pid_start(pid_list, pos);
6518}
6519
6520static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
6521{
6522	struct trace_array *tr = m->private;
6523	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
6524
6525	if (v == FTRACE_NO_PIDS)
 
6526		return NULL;
6527
6528	return trace_pid_next(pid_list, v, pos);
6529}
6530
6531static void fpid_stop(struct seq_file *m, void *p)
6532	__releases(RCU)
6533{
6534	rcu_read_unlock_sched();
6535	mutex_unlock(&ftrace_lock);
6536}
6537
6538static int fpid_show(struct seq_file *m, void *v)
6539{
6540	if (v == FTRACE_NO_PIDS) {
6541		seq_puts(m, "no pid\n");
6542		return 0;
6543	}
6544
6545	return trace_pid_show(m, v);
6546}
6547
6548static const struct seq_operations ftrace_pid_sops = {
6549	.start = fpid_start,
6550	.next = fpid_next,
6551	.stop = fpid_stop,
6552	.show = fpid_show,
6553};
6554
6555static int
6556ftrace_pid_open(struct inode *inode, struct file *file)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6557{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6558	struct trace_array *tr = inode->i_private;
6559	struct seq_file *m;
6560	int ret = 0;
6561
6562	if (trace_array_get(tr) < 0)
6563		return -ENODEV;
 
6564
6565	if ((file->f_mode & FMODE_WRITE) &&
6566	    (file->f_flags & O_TRUNC))
6567		ftrace_pid_reset(tr);
 
 
 
 
 
 
 
 
 
 
 
 
 
6568
6569	ret = seq_open(file, &ftrace_pid_sops);
6570	if (ret < 0) {
6571		trace_array_put(tr);
6572	} else {
6573		m = file->private_data;
6574		/* copy tr over to seq ops */
6575		m->private = tr;
6576	}
6577
6578	return ret;
6579}
6580
 
 
 
 
 
 
 
 
 
 
 
 
6581static void ignore_task_cpu(void *data)
6582{
6583	struct trace_array *tr = data;
6584	struct trace_pid_list *pid_list;
 
6585
6586	/*
6587	 * This function is called by on_each_cpu() while the
6588	 * event_mutex is held.
6589	 */
6590	pid_list = rcu_dereference_protected(tr->function_pids,
6591					     mutex_is_locked(&ftrace_lock));
 
 
6592
6593	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
6594		       trace_ignore_this_task(pid_list, current));
 
 
 
 
6595}
6596
6597static ssize_t
6598ftrace_pid_write(struct file *filp, const char __user *ubuf,
6599		   size_t cnt, loff_t *ppos)
6600{
6601	struct seq_file *m = filp->private_data;
6602	struct trace_array *tr = m->private;
6603	struct trace_pid_list *filtered_pids = NULL;
 
6604	struct trace_pid_list *pid_list;
6605	ssize_t ret;
6606
6607	if (!cnt)
6608		return 0;
6609
6610	mutex_lock(&ftrace_lock);
6611
6612	filtered_pids = rcu_dereference_protected(tr->function_pids,
 
 
6613					     lockdep_is_held(&ftrace_lock));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6614
6615	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
6616	if (ret < 0)
6617		goto out;
6618
6619	rcu_assign_pointer(tr->function_pids, pid_list);
 
 
 
 
 
 
 
 
6620
6621	if (filtered_pids) {
6622		synchronize_sched();
6623		trace_free_pid_list(filtered_pids);
6624	} else if (pid_list) {
6625		/* Register a probe to set whether to ignore the tracing of a task */
6626		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
6627	}
6628
6629	/*
6630	 * Ignoring of pids is done at task switch. But we have to
6631	 * check for those tasks that are currently running.
6632	 * Always do this in case a pid was appended or removed.
6633	 */
6634	on_each_cpu(ignore_task_cpu, tr, 1);
6635
6636	ftrace_update_pid_func();
6637	ftrace_startup_all(0);
6638 out:
6639	mutex_unlock(&ftrace_lock);
6640
6641	if (ret > 0)
6642		*ppos += ret;
6643
6644	return ret;
6645}
6646
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6647static int
6648ftrace_pid_release(struct inode *inode, struct file *file)
6649{
6650	struct trace_array *tr = inode->i_private;
6651
6652	trace_array_put(tr);
6653
6654	return seq_release(inode, file);
6655}
6656
6657static const struct file_operations ftrace_pid_fops = {
6658	.open		= ftrace_pid_open,
6659	.write		= ftrace_pid_write,
6660	.read		= seq_read,
6661	.llseek		= tracing_lseek,
6662	.release	= ftrace_pid_release,
6663};
6664
 
 
 
 
 
 
 
 
6665void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6666{
6667	trace_create_file("set_ftrace_pid", 0644, d_tracer,
6668			    tr, &ftrace_pid_fops);
 
 
6669}
6670
6671void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
6672					 struct dentry *d_tracer)
6673{
6674	/* Only the top level directory has the dyn_tracefs and profile */
6675	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
6676
6677	ftrace_init_dyn_tracefs(d_tracer);
6678	ftrace_profile_tracefs(d_tracer);
6679}
6680
6681/**
6682 * ftrace_kill - kill ftrace
6683 *
6684 * This function should be used by panic code. It stops ftrace
6685 * but in a not so nice way. If you need to simply kill ftrace
6686 * from a non-atomic section, use ftrace_kill.
6687 */
6688void ftrace_kill(void)
6689{
6690	ftrace_disabled = 1;
6691	ftrace_enabled = 0;
6692	clear_ftrace_function();
6693}
6694
6695/**
6696 * Test if ftrace is dead or not.
6697 */
6698int ftrace_is_dead(void)
6699{
6700	return ftrace_disabled;
6701}
6702
6703/**
6704 * register_ftrace_function - register a function for profiling
6705 * @ops - ops structure that holds the function for profiling.
6706 *
6707 * Register a function to be called by all functions in the
6708 * kernel.
6709 *
6710 * Note: @ops->func and all the functions it calls must be labeled
6711 *       with "notrace", otherwise it will go into a
6712 *       recursive loop.
6713 */
6714int register_ftrace_function(struct ftrace_ops *ops)
6715{
6716	int ret = -1;
6717
6718	ftrace_ops_init(ops);
6719
6720	mutex_lock(&ftrace_lock);
6721
6722	ret = ftrace_startup(ops, 0);
6723
6724	mutex_unlock(&ftrace_lock);
6725
6726	return ret;
6727}
6728EXPORT_SYMBOL_GPL(register_ftrace_function);
6729
6730/**
6731 * unregister_ftrace_function - unregister a function for profiling.
6732 * @ops - ops structure that holds the function to unregister
6733 *
6734 * Unregister a function that was added to be called by ftrace profiling.
6735 */
6736int unregister_ftrace_function(struct ftrace_ops *ops)
6737{
6738	int ret;
6739
6740	mutex_lock(&ftrace_lock);
6741	ret = ftrace_shutdown(ops, 0);
6742	mutex_unlock(&ftrace_lock);
6743
6744	return ret;
6745}
6746EXPORT_SYMBOL_GPL(unregister_ftrace_function);
6747
 
 
 
 
 
 
 
 
 
 
 
 
6748int
6749ftrace_enable_sysctl(struct ctl_table *table, int write,
6750		     void __user *buffer, size_t *lenp,
6751		     loff_t *ppos)
6752{
6753	int ret = -ENODEV;
6754
6755	mutex_lock(&ftrace_lock);
6756
6757	if (unlikely(ftrace_disabled))
6758		goto out;
6759
6760	ret = proc_dointvec(table, write, buffer, lenp, ppos);
6761
6762	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
6763		goto out;
6764
6765	last_ftrace_enabled = !!ftrace_enabled;
6766
6767	if (ftrace_enabled) {
6768
6769		/* we are starting ftrace again */
6770		if (rcu_dereference_protected(ftrace_ops_list,
6771			lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
6772			update_ftrace_function();
6773
6774		ftrace_startup_sysctl();
6775
6776	} else {
 
 
 
 
 
 
6777		/* stopping ftrace calls (just send to ftrace_stub) */
6778		ftrace_trace_function = ftrace_stub;
6779
6780		ftrace_shutdown_sysctl();
6781	}
6782
 
6783 out:
6784	mutex_unlock(&ftrace_lock);
6785	return ret;
6786}
6787
6788#ifdef CONFIG_FUNCTION_GRAPH_TRACER
6789
6790static struct ftrace_ops graph_ops = {
6791	.func			= ftrace_stub,
6792	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
6793				   FTRACE_OPS_FL_INITIALIZED |
6794				   FTRACE_OPS_FL_PID |
6795				   FTRACE_OPS_FL_STUB,
6796#ifdef FTRACE_GRAPH_TRAMP_ADDR
6797	.trampoline		= FTRACE_GRAPH_TRAMP_ADDR,
6798	/* trampoline_size is only needed for dynamically allocated tramps */
6799#endif
6800	ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
6801};
6802
6803void ftrace_graph_sleep_time_control(bool enable)
6804{
6805	fgraph_sleep_time = enable;
6806}
6807
6808void ftrace_graph_graph_time_control(bool enable)
6809{
6810	fgraph_graph_time = enable;
6811}
6812
6813int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
6814{
6815	return 0;
6816}
6817
6818/* The callbacks that hook a function */
6819trace_func_graph_ret_t ftrace_graph_return =
6820			(trace_func_graph_ret_t)ftrace_stub;
6821trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
6822static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
6823
6824/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
6825static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
6826{
6827	int i;
6828	int ret = 0;
6829	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
6830	struct task_struct *g, *t;
6831
6832	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
6833		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
6834					* sizeof(struct ftrace_ret_stack),
6835					GFP_KERNEL);
6836		if (!ret_stack_list[i]) {
6837			start = 0;
6838			end = i;
6839			ret = -ENOMEM;
6840			goto free;
6841		}
6842	}
6843
6844	read_lock(&tasklist_lock);
6845	do_each_thread(g, t) {
6846		if (start == end) {
6847			ret = -EAGAIN;
6848			goto unlock;
6849		}
6850
6851		if (t->ret_stack == NULL) {
6852			atomic_set(&t->tracing_graph_pause, 0);
6853			atomic_set(&t->trace_overrun, 0);
6854			t->curr_ret_stack = -1;
6855			/* Make sure the tasks see the -1 first: */
6856			smp_wmb();
6857			t->ret_stack = ret_stack_list[start++];
6858		}
6859	} while_each_thread(g, t);
6860
6861unlock:
6862	read_unlock(&tasklist_lock);
6863free:
6864	for (i = start; i < end; i++)
6865		kfree(ret_stack_list[i]);
6866	return ret;
6867}
6868
6869static void
6870ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
6871			struct task_struct *prev, struct task_struct *next)
6872{
6873	unsigned long long timestamp;
6874	int index;
6875
6876	/*
6877	 * Does the user want to count the time a function was asleep.
6878	 * If so, do not update the time stamps.
6879	 */
6880	if (fgraph_sleep_time)
6881		return;
6882
6883	timestamp = trace_clock_local();
6884
6885	prev->ftrace_timestamp = timestamp;
6886
6887	/* only process tasks that we timestamped */
6888	if (!next->ftrace_timestamp)
6889		return;
6890
6891	/*
6892	 * Update all the counters in next to make up for the
6893	 * time next was sleeping.
6894	 */
6895	timestamp -= next->ftrace_timestamp;
6896
6897	for (index = next->curr_ret_stack; index >= 0; index--)
6898		next->ret_stack[index].calltime += timestamp;
6899}
6900
6901/* Allocate a return stack for each task */
6902static int start_graph_tracing(void)
6903{
6904	struct ftrace_ret_stack **ret_stack_list;
6905	int ret, cpu;
6906
6907	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
6908				sizeof(struct ftrace_ret_stack *),
6909				GFP_KERNEL);
6910
6911	if (!ret_stack_list)
6912		return -ENOMEM;
6913
6914	/* The cpu_boot init_task->ret_stack will never be freed */
6915	for_each_online_cpu(cpu) {
6916		if (!idle_task(cpu)->ret_stack)
6917			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
6918	}
6919
6920	do {
6921		ret = alloc_retstack_tasklist(ret_stack_list);
6922	} while (ret == -EAGAIN);
6923
6924	if (!ret) {
6925		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6926		if (ret)
6927			pr_info("ftrace_graph: Couldn't activate tracepoint"
6928				" probe to kernel_sched_switch\n");
6929	}
6930
6931	kfree(ret_stack_list);
6932	return ret;
6933}
6934
6935/*
6936 * Hibernation protection.
6937 * The state of the current task is too much unstable during
6938 * suspend/restore to disk. We want to protect against that.
6939 */
6940static int
6941ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
6942							void *unused)
6943{
6944	switch (state) {
6945	case PM_HIBERNATION_PREPARE:
6946		pause_graph_tracing();
6947		break;
6948
6949	case PM_POST_HIBERNATION:
6950		unpause_graph_tracing();
6951		break;
6952	}
6953	return NOTIFY_DONE;
6954}
6955
6956static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
6957{
6958	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
6959		return 0;
6960	return __ftrace_graph_entry(trace);
6961}
6962
6963/*
6964 * The function graph tracer should only trace the functions defined
6965 * by set_ftrace_filter and set_ftrace_notrace. If another function
6966 * tracer ops is registered, the graph tracer requires testing the
6967 * function against the global ops, and not just trace any function
6968 * that any ftrace_ops registered.
6969 */
6970static void update_function_graph_func(void)
6971{
6972	struct ftrace_ops *op;
6973	bool do_test = false;
6974
6975	/*
6976	 * The graph and global ops share the same set of functions
6977	 * to test. If any other ops is on the list, then
6978	 * the graph tracing needs to test if its the function
6979	 * it should call.
6980	 */
6981	do_for_each_ftrace_op(op, ftrace_ops_list) {
6982		if (op != &global_ops && op != &graph_ops &&
6983		    op != &ftrace_list_end) {
6984			do_test = true;
6985			/* in double loop, break out with goto */
6986			goto out;
6987		}
6988	} while_for_each_ftrace_op(op);
6989 out:
6990	if (do_test)
6991		ftrace_graph_entry = ftrace_graph_entry_test;
6992	else
6993		ftrace_graph_entry = __ftrace_graph_entry;
6994}
6995
6996static struct notifier_block ftrace_suspend_notifier = {
6997	.notifier_call = ftrace_suspend_notifier_call,
6998};
6999
7000int register_ftrace_graph(trace_func_graph_ret_t retfunc,
7001			trace_func_graph_ent_t entryfunc)
7002{
7003	int ret = 0;
7004
7005	mutex_lock(&ftrace_lock);
7006
7007	/* we currently allow only one tracer registered at a time */
7008	if (ftrace_graph_active) {
7009		ret = -EBUSY;
7010		goto out;
7011	}
7012
7013	register_pm_notifier(&ftrace_suspend_notifier);
7014
7015	ftrace_graph_active++;
7016	ret = start_graph_tracing();
7017	if (ret) {
7018		ftrace_graph_active--;
7019		goto out;
7020	}
7021
7022	ftrace_graph_return = retfunc;
7023
7024	/*
7025	 * Update the indirect function to the entryfunc, and the
7026	 * function that gets called to the entry_test first. Then
7027	 * call the update fgraph entry function to determine if
7028	 * the entryfunc should be called directly or not.
7029	 */
7030	__ftrace_graph_entry = entryfunc;
7031	ftrace_graph_entry = ftrace_graph_entry_test;
7032	update_function_graph_func();
7033
7034	ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
7035out:
7036	mutex_unlock(&ftrace_lock);
7037	return ret;
7038}
7039
7040void unregister_ftrace_graph(void)
7041{
7042	mutex_lock(&ftrace_lock);
7043
7044	if (unlikely(!ftrace_graph_active))
7045		goto out;
7046
7047	ftrace_graph_active--;
7048	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
7049	ftrace_graph_entry = ftrace_graph_entry_stub;
7050	__ftrace_graph_entry = ftrace_graph_entry_stub;
7051	ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
7052	unregister_pm_notifier(&ftrace_suspend_notifier);
7053	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
7054
7055 out:
7056	mutex_unlock(&ftrace_lock);
7057}
7058
7059static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
7060
7061static void
7062graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
7063{
7064	atomic_set(&t->tracing_graph_pause, 0);
7065	atomic_set(&t->trace_overrun, 0);
7066	t->ftrace_timestamp = 0;
7067	/* make curr_ret_stack visible before we add the ret_stack */
7068	smp_wmb();
7069	t->ret_stack = ret_stack;
7070}
7071
7072/*
7073 * Allocate a return stack for the idle task. May be the first
7074 * time through, or it may be done by CPU hotplug online.
7075 */
7076void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
7077{
7078	t->curr_ret_stack = -1;
7079	/*
7080	 * The idle task has no parent, it either has its own
7081	 * stack or no stack at all.
7082	 */
7083	if (t->ret_stack)
7084		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
7085
7086	if (ftrace_graph_active) {
7087		struct ftrace_ret_stack *ret_stack;
7088
7089		ret_stack = per_cpu(idle_ret_stack, cpu);
7090		if (!ret_stack) {
7091			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
7092					    * sizeof(struct ftrace_ret_stack),
7093					    GFP_KERNEL);
7094			if (!ret_stack)
7095				return;
7096			per_cpu(idle_ret_stack, cpu) = ret_stack;
7097		}
7098		graph_init_task(t, ret_stack);
7099	}
7100}
7101
7102/* Allocate a return stack for newly created task */
7103void ftrace_graph_init_task(struct task_struct *t)
7104{
7105	/* Make sure we do not use the parent ret_stack */
7106	t->ret_stack = NULL;
7107	t->curr_ret_stack = -1;
7108
7109	if (ftrace_graph_active) {
7110		struct ftrace_ret_stack *ret_stack;
7111
7112		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
7113				* sizeof(struct ftrace_ret_stack),
7114				GFP_KERNEL);
7115		if (!ret_stack)
7116			return;
7117		graph_init_task(t, ret_stack);
7118	}
7119}
7120
7121void ftrace_graph_exit_task(struct task_struct *t)
7122{
7123	struct ftrace_ret_stack	*ret_stack = t->ret_stack;
7124
7125	t->ret_stack = NULL;
7126	/* NULL must become visible to IRQs before we free it: */
7127	barrier();
7128
7129	kfree(ret_stack);
7130}
7131#endif