1/*
   2 * Infrastructure for profiling code inserted by 'gcc -pg'.
   3 *
   4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6 *
   7 * Originally ported from the -rt patch by:
   8 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9 *
  10 * Based on code in the latency_tracer, that is:
  11 *
  12 *  Copyright (C) 2004-2006 Ingo Molnar
  13 *  Copyright (C) 2004 Nadia Yvette Chambers
  14 */
  15
  16#include <linux/stop_machine.h>
  17#include <linux/clocksource.h>
 
  18#include <linux/kallsyms.h>
 
  19#include <linux/seq_file.h>
  20#include <linux/suspend.h>
  21#include <linux/tracefs.h>
  22#include <linux/hardirq.h>
  23#include <linux/kthread.h>
  24#include <linux/uaccess.h>
  25#include <linux/bsearch.h>
  26#include <linux/module.h>
  27#include <linux/ftrace.h>
  28#include <linux/sysctl.h>
  29#include <linux/slab.h>
  30#include <linux/ctype.h>
  31#include <linux/sort.h>
  32#include <linux/list.h>
  33#include <linux/hash.h>
  34#include <linux/rcupdate.h>
 
  35
  36#include <trace/events/sched.h>
  37
 
  38#include <asm/setup.h>
  39
 
  40#include "trace_output.h"
  41#include "trace_stat.h"
  42
  43#define FTRACE_WARN_ON(cond)			\
  44	({					\
  45		int ___r = cond;		\
  46		if (WARN_ON(___r))		\
  47			ftrace_kill();		\
  48		___r;				\
  49	})
  50
  51#define FTRACE_WARN_ON_ONCE(cond)		\
  52	({					\
  53		int ___r = cond;		\
  54		if (WARN_ON_ONCE(___r))		\
  55			ftrace_kill();		\
  56		___r;				\
  57	})
  58
  59/* hash bits for specific function selection */
  60#define FTRACE_HASH_BITS 7
  61#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  62#define FTRACE_HASH_DEFAULT_BITS 10
  63#define FTRACE_HASH_MAX_BITS 12
  64
  65#ifdef CONFIG_DYNAMIC_FTRACE
  66#define INIT_OPS_HASH(opsname)	\
  67	.func_hash		= &opsname.local_hash,			\
  68	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
  69#define ASSIGN_OPS_HASH(opsname, val) \
  70	.func_hash		= val, \
  71	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
  72#else
  73#define INIT_OPS_HASH(opsname)
  74#define ASSIGN_OPS_HASH(opsname, val)
  75#endif
  76
  77static struct ftrace_ops ftrace_list_end __read_mostly = {
 
 
 
 
 
  78	.func		= ftrace_stub,
  79	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
  80	INIT_OPS_HASH(ftrace_list_end)
  81};
  82
  83/* ftrace_enabled is a method to turn ftrace on or off */
  84int ftrace_enabled __read_mostly;
  85static int last_ftrace_enabled;
  86
  87/* Current function tracing op */
  88struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
  89/* What to set function_trace_op to */
  90static struct ftrace_ops *set_function_trace_op;
  91
  92static bool ftrace_pids_enabled(struct ftrace_ops *ops)
  93{
  94	struct trace_array *tr;
  95
  96	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
  97		return false;
  98
  99	tr = ops->private;
 100
 101	return tr->function_pids != NULL;
 102}
 103
 104static void ftrace_update_trampoline(struct ftrace_ops *ops);
 105
 106/*
 107 * ftrace_disabled is set when an anomaly is discovered.
 108 * ftrace_disabled is much stronger than ftrace_enabled.
 109 */
 110static int ftrace_disabled __read_mostly;
 111
 112static DEFINE_MUTEX(ftrace_lock);
 113
 114static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
 115ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
 116static struct ftrace_ops global_ops;
 117
 118#if ARCH_SUPPORTS_FTRACE_OPS
 119static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
 120				 struct ftrace_ops *op, struct pt_regs *regs);
 121#else
 122/* See comment below, where ftrace_ops_list_func is defined */
 123static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
 124#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
 125#endif
 126
 127/*
 128 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
 129 * can use rcu_dereference_raw_notrace() is that elements removed from this list
 130 * are simply leaked, so there is no need to interact with a grace-period
 131 * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
 132 * concurrent insertions into the ftrace_global_list.
 133 *
 134 * Silly Alpha and silly pointer-speculation compiler optimizations!
 135 */
 136#define do_for_each_ftrace_op(op, list)			\
 137	op = rcu_dereference_raw_notrace(list);			\
 138	do
 139
 140/*
 141 * Optimized for just a single item in the list (as that is the normal case).
 142 */
 143#define while_for_each_ftrace_op(op)				\
 144	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
 145	       unlikely((op) != &ftrace_list_end))
 146
 147static inline void ftrace_ops_init(struct ftrace_ops *ops)
 148{
 149#ifdef CONFIG_DYNAMIC_FTRACE
 150	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
 151		mutex_init(&ops->local_hash.regex_lock);
 152		ops->func_hash = &ops->local_hash;
 153		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
 154	}
 155#endif
 156}
 157
 158/**
 159 * ftrace_nr_registered_ops - return number of ops registered
 160 *
 161 * Returns the number of ftrace_ops registered and tracing functions
 162 */
 163int ftrace_nr_registered_ops(void)
 164{
 165	struct ftrace_ops *ops;
 166	int cnt = 0;
 167
 168	mutex_lock(&ftrace_lock);
 169
 170	for (ops = ftrace_ops_list;
 171	     ops != &ftrace_list_end; ops = ops->next)
 172		cnt++;
 173
 174	mutex_unlock(&ftrace_lock);
 175
 176	return cnt;
 177}
 178
 179static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
 180			    struct ftrace_ops *op, struct pt_regs *regs)
 181{
 182	struct trace_array *tr = op->private;
 
 183
 184	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
 185		return;
 186
 187	op->saved_func(ip, parent_ip, op, regs);
 188}
 189
 190/**
 191 * clear_ftrace_function - reset the ftrace function
 192 *
 193 * This NULLs the ftrace function and in essence stops
 194 * tracing.  There may be lag
 195 */
 196void clear_ftrace_function(void)
 197{
 198	ftrace_trace_function = ftrace_stub;
 199}
 200
 201static void per_cpu_ops_disable_all(struct ftrace_ops *ops)
 202{
 203	int cpu;
 204
 205	for_each_possible_cpu(cpu)
 206		*per_cpu_ptr(ops->disabled, cpu) = 1;
 207}
 208
 209static int per_cpu_ops_alloc(struct ftrace_ops *ops)
 210{
 211	int __percpu *disabled;
 212
 213	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU)))
 214		return -EINVAL;
 215
 216	disabled = alloc_percpu(int);
 217	if (!disabled)
 218		return -ENOMEM;
 219
 220	ops->disabled = disabled;
 221	per_cpu_ops_disable_all(ops);
 222	return 0;
 223}
 224
 225static void ftrace_sync(struct work_struct *work)
 226{
 227	/*
 228	 * This function is just a stub to implement a hard force
 229	 * of synchronize_sched(). This requires synchronizing
 230	 * tasks even in userspace and idle.
 231	 *
 232	 * Yes, function tracing is rude.
 233	 */
 234}
 235
 236static void ftrace_sync_ipi(void *data)
 237{
 238	/* Probably not needed, but do it anyway */
 239	smp_rmb();
 240}
 241
 242#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 243static void update_function_graph_func(void);
 244
 245/* Both enabled by default (can be cleared by function_graph tracer flags */
 246static bool fgraph_sleep_time = true;
 247static bool fgraph_graph_time = true;
 248
 249#else
 250static inline void update_function_graph_func(void) { }
 251#endif
 252
 253
 254static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
 255{
 256	/*
 257	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
 258	 * then it needs to call the list anyway.
 259	 */
 260	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU |
 261			  FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC)
 262		return ftrace_ops_list_func;
 263
 264	return ftrace_ops_get_func(ops);
 265}
 266
 267static void update_ftrace_function(void)
 268{
 269	ftrace_func_t func;
 270
 271	/*
 272	 * Prepare the ftrace_ops that the arch callback will use.
 273	 * If there's only one ftrace_ops registered, the ftrace_ops_list
 274	 * will point to the ops we want.
 275	 */
 276	set_function_trace_op = ftrace_ops_list;
 
 277
 278	/* If there's no ftrace_ops registered, just call the stub function */
 279	if (ftrace_ops_list == &ftrace_list_end) {
 280		func = ftrace_stub;
 281
 282	/*
 283	 * If we are at the end of the list and this ops is
 284	 * recursion safe and not dynamic and the arch supports passing ops,
 285	 * then have the mcount trampoline call the function directly.
 286	 */
 287	} else if (ftrace_ops_list->next == &ftrace_list_end) {
 
 288		func = ftrace_ops_get_list_func(ftrace_ops_list);
 289
 290	} else {
 291		/* Just use the default ftrace_ops */
 292		set_function_trace_op = &ftrace_list_end;
 293		func = ftrace_ops_list_func;
 294	}
 295
 296	update_function_graph_func();
 297
 298	/* If there's no change, then do nothing more here */
 299	if (ftrace_trace_function == func)
 300		return;
 301
 302	/*
 303	 * If we are using the list function, it doesn't care
 304	 * about the function_trace_ops.
 305	 */
 306	if (func == ftrace_ops_list_func) {
 307		ftrace_trace_function = func;
 308		/*
 309		 * Don't even bother setting function_trace_ops,
 310		 * it would be racy to do so anyway.
 311		 */
 312		return;
 313	}
 314
 315#ifndef CONFIG_DYNAMIC_FTRACE
 316	/*
 317	 * For static tracing, we need to be a bit more careful.
 318	 * The function change takes affect immediately. Thus,
 319	 * we need to coorditate the setting of the function_trace_ops
 320	 * with the setting of the ftrace_trace_function.
 321	 *
 322	 * Set the function to the list ops, which will call the
 323	 * function we want, albeit indirectly, but it handles the
 324	 * ftrace_ops and doesn't depend on function_trace_op.
 325	 */
 326	ftrace_trace_function = ftrace_ops_list_func;
 327	/*
 328	 * Make sure all CPUs see this. Yes this is slow, but static
 329	 * tracing is slow and nasty to have enabled.
 330	 */
 331	schedule_on_each_cpu(ftrace_sync);
 332	/* Now all cpus are using the list ops. */
 333	function_trace_op = set_function_trace_op;
 334	/* Make sure the function_trace_op is visible on all CPUs */
 335	smp_wmb();
 336	/* Nasty way to force a rmb on all cpus */
 337	smp_call_function(ftrace_sync_ipi, NULL, 1);
 338	/* OK, we are all set to update the ftrace_trace_function now! */
 339#endif /* !CONFIG_DYNAMIC_FTRACE */
 340
 341	ftrace_trace_function = func;
 342}
 343
 344int using_ftrace_ops_list_func(void)
 
 345{
 346	return ftrace_trace_function == ftrace_ops_list_func;
 347}
 348
 349static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 350{
 351	ops->next = *list;
 352	/*
 353	 * We are entering ops into the list but another
 354	 * CPU might be walking that list. We need to make sure
 355	 * the ops->next pointer is valid before another CPU sees
 356	 * the ops pointer included into the list.
 357	 */
 358	rcu_assign_pointer(*list, ops);
 359}
 360
 361static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 
 362{
 363	struct ftrace_ops **p;
 364
 365	/*
 366	 * If we are removing the last function, then simply point
 367	 * to the ftrace_stub.
 368	 */
 369	if (*list == ops && ops->next == &ftrace_list_end) {
 
 
 
 370		*list = &ftrace_list_end;
 371		return 0;
 372	}
 373
 374	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
 375		if (*p == ops)
 376			break;
 377
 378	if (*p != ops)
 379		return -1;
 380
 381	*p = (*p)->next;
 382	return 0;
 383}
 384
 385static void ftrace_update_trampoline(struct ftrace_ops *ops);
 386
 387static int __register_ftrace_function(struct ftrace_ops *ops)
 388{
 389	if (ops->flags & FTRACE_OPS_FL_DELETED)
 390		return -EINVAL;
 391
 392	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
 393		return -EBUSY;
 394
 395#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 396	/*
 397	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
 398	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
 399	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
 400	 */
 401	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
 402	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
 403		return -EINVAL;
 404
 405	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
 406		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
 407#endif
 
 
 408
 409	if (!core_kernel_data((unsigned long)ops))
 410		ops->flags |= FTRACE_OPS_FL_DYNAMIC;
 411
 412	if (ops->flags & FTRACE_OPS_FL_PER_CPU) {
 413		if (per_cpu_ops_alloc(ops))
 414			return -ENOMEM;
 415	}
 416
 417	add_ftrace_ops(&ftrace_ops_list, ops);
 418
 419	/* Always save the function, and reset at unregistering */
 420	ops->saved_func = ops->func;
 421
 422	if (ftrace_pids_enabled(ops))
 423		ops->func = ftrace_pid_func;
 424
 425	ftrace_update_trampoline(ops);
 426
 427	if (ftrace_enabled)
 428		update_ftrace_function();
 429
 430	return 0;
 431}
 432
 433static int __unregister_ftrace_function(struct ftrace_ops *ops)
 434{
 435	int ret;
 436
 437	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
 438		return -EBUSY;
 439
 440	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
 441
 442	if (ret < 0)
 443		return ret;
 444
 445	if (ftrace_enabled)
 446		update_ftrace_function();
 447
 448	ops->func = ops->saved_func;
 449
 450	return 0;
 451}
 452
 453static void ftrace_update_pid_func(void)
 454{
 455	struct ftrace_ops *op;
 456
 457	/* Only do something if we are tracing something */
 458	if (ftrace_trace_function == ftrace_stub)
 459		return;
 460
 461	do_for_each_ftrace_op(op, ftrace_ops_list) {
 462		if (op->flags & FTRACE_OPS_FL_PID) {
 463			op->func = ftrace_pids_enabled(op) ?
 464				ftrace_pid_func : op->saved_func;
 465			ftrace_update_trampoline(op);
 466		}
 467	} while_for_each_ftrace_op(op);
 468
 469	update_ftrace_function();
 470}
 471
 472#ifdef CONFIG_FUNCTION_PROFILER
 473struct ftrace_profile {
 474	struct hlist_node		node;
 475	unsigned long			ip;
 476	unsigned long			counter;
 477#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 478	unsigned long long		time;
 479	unsigned long long		time_squared;
 480#endif
 481};
 482
 483struct ftrace_profile_page {
 484	struct ftrace_profile_page	*next;
 485	unsigned long			index;
 486	struct ftrace_profile		records[];
 487};
 488
 489struct ftrace_profile_stat {
 490	atomic_t			disabled;
 491	struct hlist_head		*hash;
 492	struct ftrace_profile_page	*pages;
 493	struct ftrace_profile_page	*start;
 494	struct tracer_stat		stat;
 495};
 496
 497#define PROFILE_RECORDS_SIZE						\
 498	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 499
 500#define PROFILES_PER_PAGE					\
 501	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 502
 503static int ftrace_profile_enabled __read_mostly;
 504
 505/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 506static DEFINE_MUTEX(ftrace_profile_lock);
 507
 508static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 509
 510#define FTRACE_PROFILE_HASH_BITS 10
 511#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
 512
 513static void *
 514function_stat_next(void *v, int idx)
 515{
 516	struct ftrace_profile *rec = v;
 517	struct ftrace_profile_page *pg;
 518
 519	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 520
 521 again:
 522	if (idx != 0)
 523		rec++;
 524
 525	if ((void *)rec >= (void *)&pg->records[pg->index]) {
 526		pg = pg->next;
 527		if (!pg)
 528			return NULL;
 529		rec = &pg->records[0];
 530		if (!rec->counter)
 531			goto again;
 532	}
 533
 534	return rec;
 535}
 536
 537static void *function_stat_start(struct tracer_stat *trace)
 538{
 539	struct ftrace_profile_stat *stat =
 540		container_of(trace, struct ftrace_profile_stat, stat);
 541
 542	if (!stat || !stat->start)
 543		return NULL;
 544
 545	return function_stat_next(&stat->start->records[0], 0);
 546}
 547
 548#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 549/* function graph compares on total time */
 550static int function_stat_cmp(void *p1, void *p2)
 551{
 552	struct ftrace_profile *a = p1;
 553	struct ftrace_profile *b = p2;
 554
 555	if (a->time < b->time)
 556		return -1;
 557	if (a->time > b->time)
 558		return 1;
 559	else
 560		return 0;
 561}
 562#else
 563/* not function graph compares against hits */
 564static int function_stat_cmp(void *p1, void *p2)
 565{
 566	struct ftrace_profile *a = p1;
 567	struct ftrace_profile *b = p2;
 568
 569	if (a->counter < b->counter)
 570		return -1;
 571	if (a->counter > b->counter)
 572		return 1;
 573	else
 574		return 0;
 575}
 576#endif
 577
 578static int function_stat_headers(struct seq_file *m)
 579{
 580#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 581	seq_puts(m, "  Function                               "
 582		 "Hit    Time            Avg             s^2\n"
 583		    "  --------                               "
 584		 "---    ----            ---             ---\n");
 585#else
 586	seq_puts(m, "  Function                               Hit\n"
 587		    "  --------                               ---\n");
 588#endif
 589	return 0;
 590}
 591
 592static int function_stat_show(struct seq_file *m, void *v)
 593{
 594	struct ftrace_profile *rec = v;
 595	char str[KSYM_SYMBOL_LEN];
 596	int ret = 0;
 597#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 598	static struct trace_seq s;
 599	unsigned long long avg;
 600	unsigned long long stddev;
 601#endif
 602	mutex_lock(&ftrace_profile_lock);
 603
 604	/* we raced with function_profile_reset() */
 605	if (unlikely(rec->counter == 0)) {
 606		ret = -EBUSY;
 607		goto out;
 608	}
 609
 610#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 611	avg = rec->time;
 612	do_div(avg, rec->counter);
 613	if (tracing_thresh && (avg < tracing_thresh))
 614		goto out;
 615#endif
 616
 617	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 618	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 619
 620#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 621	seq_puts(m, "    ");
 622
 623	/* Sample standard deviation (s^2) */
 624	if (rec->counter <= 1)
 625		stddev = 0;
 626	else {
 627		/*
 628		 * Apply Welford's method:
 629		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
 630		 */
 631		stddev = rec->counter * rec->time_squared -
 632			 rec->time * rec->time;
 633
 634		/*
 635		 * Divide only 1000 for ns^2 -> us^2 conversion.
 636		 * trace_print_graph_duration will divide 1000 again.
 637		 */
 638		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
 
 639	}
 640
 641	trace_seq_init(&s);
 642	trace_print_graph_duration(rec->time, &s);
 643	trace_seq_puts(&s, "    ");
 644	trace_print_graph_duration(avg, &s);
 645	trace_seq_puts(&s, "    ");
 646	trace_print_graph_duration(stddev, &s);
 647	trace_print_seq(m, &s);
 648#endif
 649	seq_putc(m, '\n');
 650out:
 651	mutex_unlock(&ftrace_profile_lock);
 652
 653	return ret;
 654}
 655
 656static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 657{
 658	struct ftrace_profile_page *pg;
 659
 660	pg = stat->pages = stat->start;
 661
 662	while (pg) {
 663		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 664		pg->index = 0;
 665		pg = pg->next;
 666	}
 667
 668	memset(stat->hash, 0,
 669	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 670}
 671
 672int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 673{
 674	struct ftrace_profile_page *pg;
 675	int functions;
 676	int pages;
 677	int i;
 678
 679	/* If we already allocated, do nothing */
 680	if (stat->pages)
 681		return 0;
 682
 683	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 684	if (!stat->pages)
 685		return -ENOMEM;
 686
 687#ifdef CONFIG_DYNAMIC_FTRACE
 688	functions = ftrace_update_tot_cnt;
 689#else
 690	/*
 691	 * We do not know the number of functions that exist because
 692	 * dynamic tracing is what counts them. With past experience
 693	 * we have around 20K functions. That should be more than enough.
 694	 * It is highly unlikely we will execute every function in
 695	 * the kernel.
 696	 */
 697	functions = 20000;
 698#endif
 699
 700	pg = stat->start = stat->pages;
 701
 702	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 703
 704	for (i = 1; i < pages; i++) {
 705		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 706		if (!pg->next)
 707			goto out_free;
 708		pg = pg->next;
 709	}
 710
 711	return 0;
 712
 713 out_free:
 714	pg = stat->start;
 715	while (pg) {
 716		unsigned long tmp = (unsigned long)pg;
 717
 718		pg = pg->next;
 719		free_page(tmp);
 720	}
 721
 722	stat->pages = NULL;
 723	stat->start = NULL;
 724
 725	return -ENOMEM;
 726}
 727
 728static int ftrace_profile_init_cpu(int cpu)
 729{
 730	struct ftrace_profile_stat *stat;
 731	int size;
 732
 733	stat = &per_cpu(ftrace_profile_stats, cpu);
 734
 735	if (stat->hash) {
 736		/* If the profile is already created, simply reset it */
 737		ftrace_profile_reset(stat);
 738		return 0;
 739	}
 740
 741	/*
 742	 * We are profiling all functions, but usually only a few thousand
 743	 * functions are hit. We'll make a hash of 1024 items.
 744	 */
 745	size = FTRACE_PROFILE_HASH_SIZE;
 746
 747	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 748
 749	if (!stat->hash)
 750		return -ENOMEM;
 751
 752	/* Preallocate the function profiling pages */
 753	if (ftrace_profile_pages_init(stat) < 0) {
 754		kfree(stat->hash);
 755		stat->hash = NULL;
 756		return -ENOMEM;
 757	}
 758
 759	return 0;
 760}
 761
 762static int ftrace_profile_init(void)
 763{
 764	int cpu;
 765	int ret = 0;
 766
 767	for_each_possible_cpu(cpu) {
 768		ret = ftrace_profile_init_cpu(cpu);
 769		if (ret)
 770			break;
 771	}
 772
 773	return ret;
 774}
 775
 776/* interrupts must be disabled */
 777static struct ftrace_profile *
 778ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 779{
 780	struct ftrace_profile *rec;
 781	struct hlist_head *hhd;
 782	unsigned long key;
 783
 784	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
 785	hhd = &stat->hash[key];
 786
 787	if (hlist_empty(hhd))
 788		return NULL;
 789
 790	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
 791		if (rec->ip == ip)
 792			return rec;
 793	}
 794
 795	return NULL;
 796}
 797
 798static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 799			       struct ftrace_profile *rec)
 800{
 801	unsigned long key;
 802
 803	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
 804	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 805}
 806
 807/*
 808 * The memory is already allocated, this simply finds a new record to use.
 809 */
 810static struct ftrace_profile *
 811ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 812{
 813	struct ftrace_profile *rec = NULL;
 814
 815	/* prevent recursion (from NMIs) */
 816	if (atomic_inc_return(&stat->disabled) != 1)
 817		goto out;
 818
 819	/*
 820	 * Try to find the function again since an NMI
 821	 * could have added it
 822	 */
 823	rec = ftrace_find_profiled_func(stat, ip);
 824	if (rec)
 825		goto out;
 826
 827	if (stat->pages->index == PROFILES_PER_PAGE) {
 828		if (!stat->pages->next)
 829			goto out;
 830		stat->pages = stat->pages->next;
 831	}
 832
 833	rec = &stat->pages->records[stat->pages->index++];
 834	rec->ip = ip;
 835	ftrace_add_profile(stat, rec);
 836
 837 out:
 838	atomic_dec(&stat->disabled);
 839
 840	return rec;
 841}
 842
 843static void
 844function_profile_call(unsigned long ip, unsigned long parent_ip,
 845		      struct ftrace_ops *ops, struct pt_regs *regs)
 846{
 847	struct ftrace_profile_stat *stat;
 848	struct ftrace_profile *rec;
 849	unsigned long flags;
 850
 851	if (!ftrace_profile_enabled)
 852		return;
 853
 854	local_irq_save(flags);
 855
 856	stat = this_cpu_ptr(&ftrace_profile_stats);
 857	if (!stat->hash || !ftrace_profile_enabled)
 858		goto out;
 859
 860	rec = ftrace_find_profiled_func(stat, ip);
 861	if (!rec) {
 862		rec = ftrace_profile_alloc(stat, ip);
 863		if (!rec)
 864			goto out;
 865	}
 866
 867	rec->counter++;
 868 out:
 869	local_irq_restore(flags);
 870}
 871
 872#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 
 
 
 
 
 
 
 873static int profile_graph_entry(struct ftrace_graph_ent *trace)
 874{
 875	int index = trace->depth;
 876
 877	function_profile_call(trace->func, 0, NULL, NULL);
 878
 879	if (index >= 0 && index < FTRACE_RETFUNC_DEPTH)
 880		current->ret_stack[index].subtime = 0;
 
 
 
 
 
 881
 882	return 1;
 883}
 884
 885static void profile_graph_return(struct ftrace_graph_ret *trace)
 886{
 
 887	struct ftrace_profile_stat *stat;
 888	unsigned long long calltime;
 889	struct ftrace_profile *rec;
 890	unsigned long flags;
 891
 892	local_irq_save(flags);
 893	stat = this_cpu_ptr(&ftrace_profile_stats);
 894	if (!stat->hash || !ftrace_profile_enabled)
 895		goto out;
 896
 897	/* If the calltime was zero'd ignore it */
 898	if (!trace->calltime)
 899		goto out;
 900
 901	calltime = trace->rettime - trace->calltime;
 902
 903	if (!fgraph_graph_time) {
 904		int index;
 905
 906		index = trace->depth;
 907
 908		/* Append this call time to the parent time to subtract */
 909		if (index)
 910			current->ret_stack[index - 1].subtime += calltime;
 911
 912		if (current->ret_stack[index].subtime < calltime)
 913			calltime -= current->ret_stack[index].subtime;
 
 
 914		else
 915			calltime = 0;
 916	}
 917
 918	rec = ftrace_find_profiled_func(stat, trace->func);
 919	if (rec) {
 920		rec->time += calltime;
 921		rec->time_squared += calltime * calltime;
 922	}
 923
 924 out:
 925	local_irq_restore(flags);
 926}
 927
 
 
 
 
 
 928static int register_ftrace_profiler(void)
 929{
 930	return register_ftrace_graph(&profile_graph_return,
 931				     &profile_graph_entry);
 932}
 933
 934static void unregister_ftrace_profiler(void)
 935{
 936	unregister_ftrace_graph();
 937}
 938#else
 939static struct ftrace_ops ftrace_profile_ops __read_mostly = {
 940	.func		= function_profile_call,
 941	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
 942	INIT_OPS_HASH(ftrace_profile_ops)
 943};
 944
 945static int register_ftrace_profiler(void)
 946{
 947	return register_ftrace_function(&ftrace_profile_ops);
 948}
 949
 950static void unregister_ftrace_profiler(void)
 951{
 952	unregister_ftrace_function(&ftrace_profile_ops);
 953}
 954#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 955
 956static ssize_t
 957ftrace_profile_write(struct file *filp, const char __user *ubuf,
 958		     size_t cnt, loff_t *ppos)
 959{
 960	unsigned long val;
 961	int ret;
 962
 963	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 964	if (ret)
 965		return ret;
 966
 967	val = !!val;
 968
 969	mutex_lock(&ftrace_profile_lock);
 970	if (ftrace_profile_enabled ^ val) {
 971		if (val) {
 972			ret = ftrace_profile_init();
 973			if (ret < 0) {
 974				cnt = ret;
 975				goto out;
 976			}
 977
 978			ret = register_ftrace_profiler();
 979			if (ret < 0) {
 980				cnt = ret;
 981				goto out;
 982			}
 983			ftrace_profile_enabled = 1;
 984		} else {
 985			ftrace_profile_enabled = 0;
 986			/*
 987			 * unregister_ftrace_profiler calls stop_machine
 988			 * so this acts like an synchronize_sched.
 989			 */
 990			unregister_ftrace_profiler();
 991		}
 992	}
 993 out:
 994	mutex_unlock(&ftrace_profile_lock);
 995
 996	*ppos += cnt;
 997
 998	return cnt;
 999}
1000
1001static ssize_t
1002ftrace_profile_read(struct file *filp, char __user *ubuf,
1003		     size_t cnt, loff_t *ppos)
1004{
1005	char buf[64];		/* big enough to hold a number */
1006	int r;
1007
1008	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
1009	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1010}
1011
1012static const struct file_operations ftrace_profile_fops = {
1013	.open		= tracing_open_generic,
1014	.read		= ftrace_profile_read,
1015	.write		= ftrace_profile_write,
1016	.llseek		= default_llseek,
1017};
1018
1019/* used to initialize the real stat files */
1020static struct tracer_stat function_stats __initdata = {
1021	.name		= "functions",
1022	.stat_start	= function_stat_start,
1023	.stat_next	= function_stat_next,
1024	.stat_cmp	= function_stat_cmp,
1025	.stat_headers	= function_stat_headers,
1026	.stat_show	= function_stat_show
1027};
1028
1029static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1030{
1031	struct ftrace_profile_stat *stat;
1032	struct dentry *entry;
1033	char *name;
1034	int ret;
1035	int cpu;
1036
1037	for_each_possible_cpu(cpu) {
1038		stat = &per_cpu(ftrace_profile_stats, cpu);
1039
1040		name = kasprintf(GFP_KERNEL, "function%d", cpu);
1041		if (!name) {
1042			/*
1043			 * The files created are permanent, if something happens
1044			 * we still do not free memory.
1045			 */
1046			WARN(1,
1047			     "Could not allocate stat file for cpu %d\n",
1048			     cpu);
1049			return;
1050		}
1051		stat->stat = function_stats;
1052		stat->stat.name = name;
1053		ret = register_stat_tracer(&stat->stat);
1054		if (ret) {
1055			WARN(1,
1056			     "Could not register function stat for cpu %d\n",
1057			     cpu);
1058			kfree(name);
1059			return;
1060		}
1061	}
1062
1063	entry = tracefs_create_file("function_profile_enabled", 0644,
1064				    d_tracer, NULL, &ftrace_profile_fops);
1065	if (!entry)
1066		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
1067}
1068
1069#else /* CONFIG_FUNCTION_PROFILER */
1070static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1071{
1072}
1073#endif /* CONFIG_FUNCTION_PROFILER */
1074
1075static struct pid * const ftrace_swapper_pid = &init_struct_pid;
1076
1077#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1078static int ftrace_graph_active;
1079#else
1080# define ftrace_graph_active 0
1081#endif
1082
1083#ifdef CONFIG_DYNAMIC_FTRACE
1084
1085static struct ftrace_ops *removed_ops;
1086
1087/*
1088 * Set when doing a global update, like enabling all recs or disabling them.
1089 * It is not set when just updating a single ftrace_ops.
1090 */
1091static bool update_all_ops;
1092
1093#ifndef CONFIG_FTRACE_MCOUNT_RECORD
1094# error Dynamic ftrace depends on MCOUNT_RECORD
1095#endif
1096
1097static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
1098
1099struct ftrace_func_probe {
1100	struct hlist_node	node;
1101	struct ftrace_probe_ops	*ops;
1102	unsigned long		flags;
1103	unsigned long		ip;
1104	void			*data;
1105	struct list_head	free_list;
1106};
1107
1108struct ftrace_func_entry {
1109	struct hlist_node hlist;
1110	unsigned long ip;
1111};
1112
1113struct ftrace_hash {
1114	unsigned long		size_bits;
1115	struct hlist_head	*buckets;
1116	unsigned long		count;
1117	struct rcu_head		rcu;
1118};
1119
1120/*
1121 * We make these constant because no one should touch them,
1122 * but they are used as the default "empty hash", to avoid allocating
1123 * it all the time. These are in a read only section such that if
1124 * anyone does try to modify it, it will cause an exception.
1125 */
1126static const struct hlist_head empty_buckets[1];
1127static const struct ftrace_hash empty_hash = {
1128	.buckets = (struct hlist_head *)empty_buckets,
1129};
1130#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1131
1132static struct ftrace_ops global_ops = {
1133	.func				= ftrace_stub,
1134	.local_hash.notrace_hash	= EMPTY_HASH,
1135	.local_hash.filter_hash		= EMPTY_HASH,
1136	INIT_OPS_HASH(global_ops)
1137	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1138					  FTRACE_OPS_FL_INITIALIZED |
1139					  FTRACE_OPS_FL_PID,
1140};
1141
1142/*
1143 * This is used by __kernel_text_address() to return true if the
1144 * address is on a dynamically allocated trampoline that would
1145 * not return true for either core_kernel_text() or
1146 * is_module_text_address().
1147 */
1148bool is_ftrace_trampoline(unsigned long addr)
1149{
1150	struct ftrace_ops *op;
1151	bool ret = false;
1152
1153	/*
1154	 * Some of the ops may be dynamically allocated,
1155	 * they are freed after a synchronize_sched().
1156	 */
1157	preempt_disable_notrace();
1158
1159	do_for_each_ftrace_op(op, ftrace_ops_list) {
1160		/*
1161		 * This is to check for dynamically allocated trampolines.
1162		 * Trampolines that are in kernel text will have
1163		 * core_kernel_text() return true.
1164		 */
1165		if (op->trampoline && op->trampoline_size)
1166			if (addr >= op->trampoline &&
1167			    addr < op->trampoline + op->trampoline_size) {
1168				ret = true;
1169				goto out;
1170			}
1171	} while_for_each_ftrace_op(op);
1172
1173 out:
1174	preempt_enable_notrace();
1175
1176	return ret;
 
 
 
 
 
 
 
 
 
 
 
1177}
1178
1179struct ftrace_page {
1180	struct ftrace_page	*next;
1181	struct dyn_ftrace	*records;
1182	int			index;
1183	int			size;
1184};
1185
1186#define ENTRY_SIZE sizeof(struct dyn_ftrace)
1187#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1188
1189/* estimate from running different kernels */
1190#define NR_TO_INIT		10000
1191
1192static struct ftrace_page	*ftrace_pages_start;
1193static struct ftrace_page	*ftrace_pages;
1194
1195static bool __always_inline ftrace_hash_empty(struct ftrace_hash *hash)
 
1196{
1197	return !hash || !hash->count;
 
 
 
1198}
1199
1200static struct ftrace_func_entry *
1201ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
 
1202{
1203	unsigned long key;
1204	struct ftrace_func_entry *entry;
1205	struct hlist_head *hhd;
1206
1207	if (ftrace_hash_empty(hash))
1208		return NULL;
1209
1210	if (hash->size_bits > 0)
1211		key = hash_long(ip, hash->size_bits);
1212	else
1213		key = 0;
1214
1215	hhd = &hash->buckets[key];
1216
1217	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1218		if (entry->ip == ip)
1219			return entry;
1220	}
1221	return NULL;
1222}
1223
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1224static void __add_hash_entry(struct ftrace_hash *hash,
1225			     struct ftrace_func_entry *entry)
1226{
1227	struct hlist_head *hhd;
1228	unsigned long key;
1229
1230	if (hash->size_bits)
1231		key = hash_long(entry->ip, hash->size_bits);
1232	else
1233		key = 0;
1234
1235	hhd = &hash->buckets[key];
1236	hlist_add_head(&entry->hlist, hhd);
1237	hash->count++;
1238}
1239
1240static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1241{
1242	struct ftrace_func_entry *entry;
1243
1244	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1245	if (!entry)
1246		return -ENOMEM;
1247
1248	entry->ip = ip;
1249	__add_hash_entry(hash, entry);
1250
1251	return 0;
1252}
1253
1254static void
1255free_hash_entry(struct ftrace_hash *hash,
1256		  struct ftrace_func_entry *entry)
1257{
1258	hlist_del(&entry->hlist);
1259	kfree(entry);
1260	hash->count--;
1261}
1262
1263static void
1264remove_hash_entry(struct ftrace_hash *hash,
1265		  struct ftrace_func_entry *entry)
1266{
1267	hlist_del(&entry->hlist);
1268	hash->count--;
1269}
1270
1271static void ftrace_hash_clear(struct ftrace_hash *hash)
1272{
1273	struct hlist_head *hhd;
1274	struct hlist_node *tn;
1275	struct ftrace_func_entry *entry;
1276	int size = 1 << hash->size_bits;
1277	int i;
1278
1279	if (!hash->count)
1280		return;
1281
1282	for (i = 0; i < size; i++) {
1283		hhd = &hash->buckets[i];
1284		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1285			free_hash_entry(hash, entry);
1286	}
1287	FTRACE_WARN_ON(hash->count);
1288}
1289
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1290static void free_ftrace_hash(struct ftrace_hash *hash)
1291{
1292	if (!hash || hash == EMPTY_HASH)
1293		return;
1294	ftrace_hash_clear(hash);
1295	kfree(hash->buckets);
1296	kfree(hash);
1297}
1298
1299static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1300{
1301	struct ftrace_hash *hash;
1302
1303	hash = container_of(rcu, struct ftrace_hash, rcu);
1304	free_ftrace_hash(hash);
1305}
1306
1307static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1308{
1309	if (!hash || hash == EMPTY_HASH)
1310		return;
1311	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1312}
1313
1314void ftrace_free_filter(struct ftrace_ops *ops)
1315{
1316	ftrace_ops_init(ops);
1317	free_ftrace_hash(ops->func_hash->filter_hash);
1318	free_ftrace_hash(ops->func_hash->notrace_hash);
1319}
1320
1321static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1322{
1323	struct ftrace_hash *hash;
1324	int size;
1325
1326	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1327	if (!hash)
1328		return NULL;
1329
1330	size = 1 << size_bits;
1331	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1332
1333	if (!hash->buckets) {
1334		kfree(hash);
1335		return NULL;
1336	}
1337
1338	hash->size_bits = size_bits;
1339
1340	return hash;
1341}
1342
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1343static struct ftrace_hash *
1344alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1345{
1346	struct ftrace_func_entry *entry;
1347	struct ftrace_hash *new_hash;
1348	int size;
1349	int ret;
1350	int i;
1351
1352	new_hash = alloc_ftrace_hash(size_bits);
1353	if (!new_hash)
1354		return NULL;
1355
 
 
 
1356	/* Empty hash? */
1357	if (ftrace_hash_empty(hash))
1358		return new_hash;
1359
1360	size = 1 << hash->size_bits;
1361	for (i = 0; i < size; i++) {
1362		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1363			ret = add_hash_entry(new_hash, entry->ip);
1364			if (ret < 0)
1365				goto free_hash;
1366		}
1367	}
1368
1369	FTRACE_WARN_ON(new_hash->count != hash->count);
1370
1371	return new_hash;
1372
1373 free_hash:
1374	free_ftrace_hash(new_hash);
1375	return NULL;
1376}
1377
1378static void
1379ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1380static void
1381ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1382
1383static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1384				       struct ftrace_hash *new_hash);
1385
1386static int
1387ftrace_hash_move(struct ftrace_ops *ops, int enable,
1388		 struct ftrace_hash **dst, struct ftrace_hash *src)
1389{
1390	struct ftrace_func_entry *entry;
1391	struct hlist_node *tn;
1392	struct hlist_head *hhd;
1393	struct ftrace_hash *new_hash;
1394	int size = src->count;
 
1395	int bits = 0;
1396	int ret;
1397	int i;
1398
1399	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
1400	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
1401		return -EINVAL;
1402
1403	/*
1404	 * If the new source is empty, just free dst and assign it
1405	 * the empty_hash.
1406	 */
1407	if (!src->count) {
1408		new_hash = EMPTY_HASH;
1409		goto update;
1410	}
1411
1412	/*
1413	 * Make the hash size about 1/2 the # found
1414	 */
1415	for (size /= 2; size; size >>= 1)
1416		bits++;
1417
1418	/* Don't allocate too much */
1419	if (bits > FTRACE_HASH_MAX_BITS)
1420		bits = FTRACE_HASH_MAX_BITS;
1421
1422	new_hash = alloc_ftrace_hash(bits);
1423	if (!new_hash)
1424		return -ENOMEM;
 
 
1425
1426	size = 1 << src->size_bits;
1427	for (i = 0; i < size; i++) {
1428		hhd = &src->buckets[i];
1429		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1430			remove_hash_entry(src, entry);
1431			__add_hash_entry(new_hash, entry);
1432		}
1433	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1434
1435update:
1436	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
1437	if (enable) {
1438		/* IPMODIFY should be updated only when filter_hash updating */
1439		ret = ftrace_hash_ipmodify_update(ops, new_hash);
1440		if (ret < 0) {
1441			free_ftrace_hash(new_hash);
1442			return ret;
1443		}
1444	}
1445
1446	/*
1447	 * Remove the current set, update the hash and add
1448	 * them back.
1449	 */
1450	ftrace_hash_rec_disable_modify(ops, enable);
1451
1452	rcu_assign_pointer(*dst, new_hash);
1453
1454	ftrace_hash_rec_enable_modify(ops, enable);
1455
1456	return 0;
1457}
1458
1459static bool hash_contains_ip(unsigned long ip,
1460			     struct ftrace_ops_hash *hash)
1461{
1462	/*
1463	 * The function record is a match if it exists in the filter
1464	 * hash and not in the notrace hash. Note, an emty hash is
1465	 * considered a match for the filter hash, but an empty
1466	 * notrace hash is considered not in the notrace hash.
1467	 */
1468	return (ftrace_hash_empty(hash->filter_hash) ||
1469		ftrace_lookup_ip(hash->filter_hash, ip)) &&
1470		(ftrace_hash_empty(hash->notrace_hash) ||
1471		 !ftrace_lookup_ip(hash->notrace_hash, ip));
1472}
1473
1474/*
1475 * Test the hashes for this ops to see if we want to call
1476 * the ops->func or not.
1477 *
1478 * It's a match if the ip is in the ops->filter_hash or
1479 * the filter_hash does not exist or is empty,
1480 *  AND
1481 * the ip is not in the ops->notrace_hash.
1482 *
1483 * This needs to be called with preemption disabled as
1484 * the hashes are freed with call_rcu_sched().
1485 */
1486static int
1487ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1488{
1489	struct ftrace_ops_hash hash;
1490	int ret;
1491
1492#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1493	/*
1494	 * There's a small race when adding ops that the ftrace handler
1495	 * that wants regs, may be called without them. We can not
1496	 * allow that handler to be called if regs is NULL.
1497	 */
1498	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1499		return 0;
1500#endif
1501
1502	hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);
1503	hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);
1504
1505	if (hash_contains_ip(ip, &hash))
1506		ret = 1;
1507	else
1508		ret = 0;
1509
1510	return ret;
1511}
1512
1513/*
1514 * This is a double for. Do not use 'break' to break out of the loop,
1515 * you must use a goto.
1516 */
1517#define do_for_each_ftrace_rec(pg, rec)					\
1518	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1519		int _____i;						\
1520		for (_____i = 0; _____i < pg->index; _____i++) {	\
1521			rec = &pg->records[_____i];
1522
1523#define while_for_each_ftrace_rec()		\
1524		}				\
1525	}
1526
1527
1528static int ftrace_cmp_recs(const void *a, const void *b)
1529{
1530	const struct dyn_ftrace *key = a;
1531	const struct dyn_ftrace *rec = b;
1532
1533	if (key->flags < rec->ip)
1534		return -1;
1535	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1536		return 1;
1537	return 0;
1538}
1539
1540/**
1541 * ftrace_location_range - return the first address of a traced location
1542 *	if it touches the given ip range
1543 * @start: start of range to search.
1544 * @end: end of range to search (inclusive). @end points to the last byte
1545 *	to check.
1546 *
1547 * Returns rec->ip if the related ftrace location is a least partly within
1548 * the given address range. That is, the first address of the instruction
1549 * that is either a NOP or call to the function tracer. It checks the ftrace
1550 * internal tables to determine if the address belongs or not.
1551 */
1552unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1553{
1554	struct ftrace_page *pg;
1555	struct dyn_ftrace *rec;
1556	struct dyn_ftrace key;
1557
1558	key.ip = start;
1559	key.flags = end;	/* overload flags, as it is unsigned long */
1560
1561	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1562		if (end < pg->records[0].ip ||
1563		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1564			continue;
1565		rec = bsearch(&key, pg->records, pg->index,
1566			      sizeof(struct dyn_ftrace),
1567			      ftrace_cmp_recs);
1568		if (rec)
1569			return rec->ip;
1570	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1571
1572	return 0;
1573}
1574
1575/**
1576 * ftrace_location - return true if the ip giving is a traced location
1577 * @ip: the instruction pointer to check
1578 *
1579 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1580 * That is, the instruction that is either a NOP or call to
1581 * the function tracer. It checks the ftrace internal tables to
1582 * determine if the address belongs or not.
1583 */
1584unsigned long ftrace_location(unsigned long ip)
1585{
1586	return ftrace_location_range(ip, ip);
1587}
1588
1589/**
1590 * ftrace_text_reserved - return true if range contains an ftrace location
1591 * @start: start of range to search
1592 * @end: end of range to search (inclusive). @end points to the last byte to check.
1593 *
1594 * Returns 1 if @start and @end contains a ftrace location.
1595 * That is, the instruction that is either a NOP or call to
1596 * the function tracer. It checks the ftrace internal tables to
1597 * determine if the address belongs or not.
1598 */
1599int ftrace_text_reserved(const void *start, const void *end)
1600{
1601	unsigned long ret;
1602
1603	ret = ftrace_location_range((unsigned long)start,
1604				    (unsigned long)end);
1605
1606	return (int)!!ret;
1607}
1608
1609/* Test if ops registered to this rec needs regs */
1610static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
1611{
1612	struct ftrace_ops *ops;
1613	bool keep_regs = false;
1614
1615	for (ops = ftrace_ops_list;
1616	     ops != &ftrace_list_end; ops = ops->next) {
1617		/* pass rec in as regs to have non-NULL val */
1618		if (ftrace_ops_test(ops, rec->ip, rec)) {
1619			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1620				keep_regs = true;
1621				break;
1622			}
1623		}
1624	}
1625
1626	return  keep_regs;
1627}
1628
 
 
 
 
 
 
 
1629static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1630				     int filter_hash,
1631				     bool inc)
1632{
1633	struct ftrace_hash *hash;
1634	struct ftrace_hash *other_hash;
1635	struct ftrace_page *pg;
1636	struct dyn_ftrace *rec;
1637	bool update = false;
1638	int count = 0;
1639	int all = 0;
1640
1641	/* Only update if the ops has been registered */
1642	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1643		return false;
1644
1645	/*
1646	 * In the filter_hash case:
1647	 *   If the count is zero, we update all records.
1648	 *   Otherwise we just update the items in the hash.
1649	 *
1650	 * In the notrace_hash case:
1651	 *   We enable the update in the hash.
1652	 *   As disabling notrace means enabling the tracing,
1653	 *   and enabling notrace means disabling, the inc variable
1654	 *   gets inversed.
1655	 */
1656	if (filter_hash) {
1657		hash = ops->func_hash->filter_hash;
1658		other_hash = ops->func_hash->notrace_hash;
1659		if (ftrace_hash_empty(hash))
1660			all = 1;
1661	} else {
1662		inc = !inc;
1663		hash = ops->func_hash->notrace_hash;
1664		other_hash = ops->func_hash->filter_hash;
1665		/*
1666		 * If the notrace hash has no items,
1667		 * then there's nothing to do.
1668		 */
1669		if (ftrace_hash_empty(hash))
1670			return false;
1671	}
1672
1673	do_for_each_ftrace_rec(pg, rec) {
1674		int in_other_hash = 0;
1675		int in_hash = 0;
1676		int match = 0;
1677
1678		if (rec->flags & FTRACE_FL_DISABLED)
1679			continue;
1680
1681		if (all) {
1682			/*
1683			 * Only the filter_hash affects all records.
1684			 * Update if the record is not in the notrace hash.
1685			 */
1686			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1687				match = 1;
1688		} else {
1689			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1690			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1691
1692			/*
1693			 * If filter_hash is set, we want to match all functions
1694			 * that are in the hash but not in the other hash.
1695			 *
1696			 * If filter_hash is not set, then we are decrementing.
1697			 * That means we match anything that is in the hash
1698			 * and also in the other_hash. That is, we need to turn
1699			 * off functions in the other hash because they are disabled
1700			 * by this hash.
1701			 */
1702			if (filter_hash && in_hash && !in_other_hash)
1703				match = 1;
1704			else if (!filter_hash && in_hash &&
1705				 (in_other_hash || ftrace_hash_empty(other_hash)))
1706				match = 1;
1707		}
1708		if (!match)
1709			continue;
1710
1711		if (inc) {
1712			rec->flags++;
1713			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1714				return false;
1715
 
 
 
1716			/*
1717			 * If there's only a single callback registered to a
1718			 * function, and the ops has a trampoline registered
1719			 * for it, then we can call it directly.
1720			 */
1721			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1722				rec->flags |= FTRACE_FL_TRAMP;
1723			else
1724				/*
1725				 * If we are adding another function callback
1726				 * to this function, and the previous had a
1727				 * custom trampoline in use, then we need to go
1728				 * back to the default trampoline.
1729				 */
1730				rec->flags &= ~FTRACE_FL_TRAMP;
1731
1732			/*
1733			 * If any ops wants regs saved for this function
1734			 * then all ops will get saved regs.
1735			 */
1736			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1737				rec->flags |= FTRACE_FL_REGS;
1738		} else {
1739			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1740				return false;
1741			rec->flags--;
1742
1743			/*
 
 
 
 
 
 
 
 
 
1744			 * If the rec had REGS enabled and the ops that is
1745			 * being removed had REGS set, then see if there is
1746			 * still any ops for this record that wants regs.
1747			 * If not, we can stop recording them.
1748			 */
1749			if (ftrace_rec_count(rec) > 0 &&
1750			    rec->flags & FTRACE_FL_REGS &&
1751			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1752				if (!test_rec_ops_needs_regs(rec))
1753					rec->flags &= ~FTRACE_FL_REGS;
1754			}
1755
1756			/*
1757			 * If the rec had TRAMP enabled, then it needs to
1758			 * be cleared. As TRAMP can only be enabled iff
1759			 * there is only a single ops attached to it.
1760			 * In otherwords, always disable it on decrementing.
1761			 * In the future, we may set it if rec count is
1762			 * decremented to one, and the ops that is left
1763			 * has a trampoline.
1764			 */
1765			rec->flags &= ~FTRACE_FL_TRAMP;
 
 
 
 
1766
1767			/*
1768			 * flags will be cleared in ftrace_check_record()
1769			 * if rec count is zero.
1770			 */
1771		}
1772		count++;
1773
1774		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
1775		update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;
1776
1777		/* Shortcut, if we handled all records, we are done. */
1778		if (!all && count == hash->count)
1779			return update;
1780	} while_for_each_ftrace_rec();
1781
1782	return update;
1783}
1784
1785static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1786				    int filter_hash)
1787{
1788	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1789}
1790
1791static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1792				   int filter_hash)
1793{
1794	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1795}
1796
1797static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
1798					  int filter_hash, int inc)
1799{
1800	struct ftrace_ops *op;
1801
1802	__ftrace_hash_rec_update(ops, filter_hash, inc);
1803
1804	if (ops->func_hash != &global_ops.local_hash)
1805		return;
1806
1807	/*
1808	 * If the ops shares the global_ops hash, then we need to update
1809	 * all ops that are enabled and use this hash.
1810	 */
1811	do_for_each_ftrace_op(op, ftrace_ops_list) {
1812		/* Already done */
1813		if (op == ops)
1814			continue;
1815		if (op->func_hash == &global_ops.local_hash)
1816			__ftrace_hash_rec_update(op, filter_hash, inc);
1817	} while_for_each_ftrace_op(op);
1818}
1819
1820static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
1821					   int filter_hash)
1822{
1823	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
1824}
1825
1826static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
1827					  int filter_hash)
1828{
1829	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
1830}
1831
1832/*
1833 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
1834 * or no-needed to update, -EBUSY if it detects a conflict of the flag
1835 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1836 * Note that old_hash and new_hash has below meanings
1837 *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1838 *  - If the hash is EMPTY_HASH, it hits nothing
1839 *  - Anything else hits the recs which match the hash entries.
1840 */
1841static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
1842					 struct ftrace_hash *old_hash,
1843					 struct ftrace_hash *new_hash)
1844{
1845	struct ftrace_page *pg;
1846	struct dyn_ftrace *rec, *end = NULL;
1847	int in_old, in_new;
1848
1849	/* Only update if the ops has been registered */
1850	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1851		return 0;
1852
1853	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
1854		return 0;
1855
1856	/*
1857	 * Since the IPMODIFY is a very address sensitive action, we do not
1858	 * allow ftrace_ops to set all functions to new hash.
1859	 */
1860	if (!new_hash || !old_hash)
1861		return -EINVAL;
1862
1863	/* Update rec->flags */
1864	do_for_each_ftrace_rec(pg, rec) {
1865
1866		if (rec->flags & FTRACE_FL_DISABLED)
1867			continue;
1868
1869		/* We need to update only differences of filter_hash */
1870		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1871		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1872		if (in_old == in_new)
1873			continue;
1874
1875		if (in_new) {
1876			/* New entries must ensure no others are using it */
1877			if (rec->flags & FTRACE_FL_IPMODIFY)
1878				goto rollback;
1879			rec->flags |= FTRACE_FL_IPMODIFY;
1880		} else /* Removed entry */
1881			rec->flags &= ~FTRACE_FL_IPMODIFY;
1882	} while_for_each_ftrace_rec();
1883
1884	return 0;
1885
1886rollback:
1887	end = rec;
1888
1889	/* Roll back what we did above */
1890	do_for_each_ftrace_rec(pg, rec) {
1891
1892		if (rec->flags & FTRACE_FL_DISABLED)
1893			continue;
1894
1895		if (rec == end)
1896			goto err_out;
1897
1898		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1899		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1900		if (in_old == in_new)
1901			continue;
1902
1903		if (in_new)
1904			rec->flags &= ~FTRACE_FL_IPMODIFY;
1905		else
1906			rec->flags |= FTRACE_FL_IPMODIFY;
1907	} while_for_each_ftrace_rec();
1908
1909err_out:
1910	return -EBUSY;
1911}
1912
1913static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
1914{
1915	struct ftrace_hash *hash = ops->func_hash->filter_hash;
1916
1917	if (ftrace_hash_empty(hash))
1918		hash = NULL;
1919
1920	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
1921}
1922
1923/* Disabling always succeeds */
1924static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
1925{
1926	struct ftrace_hash *hash = ops->func_hash->filter_hash;
1927
1928	if (ftrace_hash_empty(hash))
1929		hash = NULL;
1930
1931	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
1932}
1933
1934static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1935				       struct ftrace_hash *new_hash)
1936{
1937	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
1938
1939	if (ftrace_hash_empty(old_hash))
1940		old_hash = NULL;
1941
1942	if (ftrace_hash_empty(new_hash))
1943		new_hash = NULL;
1944
1945	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
1946}
1947
1948static void print_ip_ins(const char *fmt, const unsigned char *p)
1949{
 
1950	int i;
1951
 
 
 
 
 
1952	printk(KERN_CONT "%s", fmt);
1953
1954	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1955		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1956}
1957
1958static struct ftrace_ops *
1959ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1960static struct ftrace_ops *
1961ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1962
1963enum ftrace_bug_type ftrace_bug_type;
1964const void *ftrace_expected;
1965
1966static void print_bug_type(void)
1967{
1968	switch (ftrace_bug_type) {
1969	case FTRACE_BUG_UNKNOWN:
1970		break;
1971	case FTRACE_BUG_INIT:
1972		pr_info("Initializing ftrace call sites\n");
1973		break;
1974	case FTRACE_BUG_NOP:
1975		pr_info("Setting ftrace call site to NOP\n");
1976		break;
1977	case FTRACE_BUG_CALL:
1978		pr_info("Setting ftrace call site to call ftrace function\n");
1979		break;
1980	case FTRACE_BUG_UPDATE:
1981		pr_info("Updating ftrace call site to call a different ftrace function\n");
1982		break;
1983	}
1984}
1985
1986/**
1987 * ftrace_bug - report and shutdown function tracer
1988 * @failed: The failed type (EFAULT, EINVAL, EPERM)
1989 * @rec: The record that failed
1990 *
1991 * The arch code that enables or disables the function tracing
1992 * can call ftrace_bug() when it has detected a problem in
1993 * modifying the code. @failed should be one of either:
1994 * EFAULT - if the problem happens on reading the @ip address
1995 * EINVAL - if what is read at @ip is not what was expected
1996 * EPERM - if the problem happens on writting to the @ip address
1997 */
1998void ftrace_bug(int failed, struct dyn_ftrace *rec)
1999{
2000	unsigned long ip = rec ? rec->ip : 0;
2001
 
 
2002	switch (failed) {
2003	case -EFAULT:
2004		FTRACE_WARN_ON_ONCE(1);
2005		pr_info("ftrace faulted on modifying ");
2006		print_ip_sym(ip);
2007		break;
2008	case -EINVAL:
2009		FTRACE_WARN_ON_ONCE(1);
2010		pr_info("ftrace failed to modify ");
2011		print_ip_sym(ip);
2012		print_ip_ins(" actual:   ", (unsigned char *)ip);
2013		pr_cont("\n");
2014		if (ftrace_expected) {
2015			print_ip_ins(" expected: ", ftrace_expected);
2016			pr_cont("\n");
2017		}
2018		break;
2019	case -EPERM:
2020		FTRACE_WARN_ON_ONCE(1);
2021		pr_info("ftrace faulted on writing ");
2022		print_ip_sym(ip);
2023		break;
2024	default:
2025		FTRACE_WARN_ON_ONCE(1);
2026		pr_info("ftrace faulted on unknown error ");
2027		print_ip_sym(ip);
2028	}
2029	print_bug_type();
2030	if (rec) {
2031		struct ftrace_ops *ops = NULL;
2032
2033		pr_info("ftrace record flags: %lx\n", rec->flags);
2034		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
2035			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
2036		if (rec->flags & FTRACE_FL_TRAMP_EN) {
2037			ops = ftrace_find_tramp_ops_any(rec);
2038			if (ops) {
2039				do {
2040					pr_cont("\ttramp: %pS (%pS)",
2041						(void *)ops->trampoline,
2042						(void *)ops->func);
2043					ops = ftrace_find_tramp_ops_next(rec, ops);
2044				} while (ops);
2045			} else
2046				pr_cont("\ttramp: ERROR!");
2047
2048		}
2049		ip = ftrace_get_addr_curr(rec);
2050		pr_cont("\n expected tramp: %lx\n", ip);
2051	}
 
 
2052}
2053
2054static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
2055{
2056	unsigned long flag = 0UL;
2057
2058	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2059
2060	if (rec->flags & FTRACE_FL_DISABLED)
2061		return FTRACE_UPDATE_IGNORE;
2062
2063	/*
2064	 * If we are updating calls:
2065	 *
2066	 *   If the record has a ref count, then we need to enable it
2067	 *   because someone is using it.
2068	 *
2069	 *   Otherwise we make sure its disabled.
2070	 *
2071	 * If we are disabling calls, then disable all records that
2072	 * are enabled.
2073	 */
2074	if (enable && ftrace_rec_count(rec))
2075		flag = FTRACE_FL_ENABLED;
2076
2077	/*
2078	 * If enabling and the REGS flag does not match the REGS_EN, or
2079	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
2080	 * this record. Set flags to fail the compare against ENABLED.
 
2081	 */
2082	if (flag) {
2083		if (!(rec->flags & FTRACE_FL_REGS) != 
2084		    !(rec->flags & FTRACE_FL_REGS_EN))
2085			flag |= FTRACE_FL_REGS;
2086
2087		if (!(rec->flags & FTRACE_FL_TRAMP) != 
2088		    !(rec->flags & FTRACE_FL_TRAMP_EN))
2089			flag |= FTRACE_FL_TRAMP;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2090	}
2091
2092	/* If the state of this record hasn't changed, then do nothing */
2093	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2094		return FTRACE_UPDATE_IGNORE;
2095
2096	if (flag) {
2097		/* Save off if rec is being enabled (for return value) */
2098		flag ^= rec->flags & FTRACE_FL_ENABLED;
2099
2100		if (update) {
2101			rec->flags |= FTRACE_FL_ENABLED;
2102			if (flag & FTRACE_FL_REGS) {
2103				if (rec->flags & FTRACE_FL_REGS)
2104					rec->flags |= FTRACE_FL_REGS_EN;
2105				else
2106					rec->flags &= ~FTRACE_FL_REGS_EN;
2107			}
2108			if (flag & FTRACE_FL_TRAMP) {
2109				if (rec->flags & FTRACE_FL_TRAMP)
2110					rec->flags |= FTRACE_FL_TRAMP_EN;
2111				else
2112					rec->flags &= ~FTRACE_FL_TRAMP_EN;
2113			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2114		}
2115
2116		/*
2117		 * If this record is being updated from a nop, then
2118		 *   return UPDATE_MAKE_CALL.
2119		 * Otherwise,
2120		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2121		 *   from the save regs, to a non-save regs function or
2122		 *   vice versa, or from a trampoline call.
2123		 */
2124		if (flag & FTRACE_FL_ENABLED) {
2125			ftrace_bug_type = FTRACE_BUG_CALL;
2126			return FTRACE_UPDATE_MAKE_CALL;
2127		}
2128
2129		ftrace_bug_type = FTRACE_BUG_UPDATE;
2130		return FTRACE_UPDATE_MODIFY_CALL;
2131	}
2132
2133	if (update) {
2134		/* If there's no more users, clear all flags */
2135		if (!ftrace_rec_count(rec))
2136			rec->flags = 0;
2137		else
2138			/*
2139			 * Just disable the record, but keep the ops TRAMP
2140			 * and REGS states. The _EN flags must be disabled though.
2141			 */
2142			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
2143					FTRACE_FL_REGS_EN);
2144	}
2145
2146	ftrace_bug_type = FTRACE_BUG_NOP;
2147	return FTRACE_UPDATE_MAKE_NOP;
2148}
2149
2150/**
2151 * ftrace_update_record, set a record that now is tracing or not
2152 * @rec: the record to update
2153 * @enable: set to 1 if the record is tracing, zero to force disable
2154 *
2155 * The records that represent all functions that can be traced need
2156 * to be updated when tracing has been enabled.
2157 */
2158int ftrace_update_record(struct dyn_ftrace *rec, int enable)
2159{
2160	return ftrace_check_record(rec, enable, 1);
2161}
2162
2163/**
2164 * ftrace_test_record, check if the record has been enabled or not
2165 * @rec: the record to test
2166 * @enable: set to 1 to check if enabled, 0 if it is disabled
2167 *
2168 * The arch code may need to test if a record is already set to
2169 * tracing to determine how to modify the function code that it
2170 * represents.
2171 */
2172int ftrace_test_record(struct dyn_ftrace *rec, int enable)
2173{
2174	return ftrace_check_record(rec, enable, 0);
2175}
2176
2177static struct ftrace_ops *
2178ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
2179{
2180	struct ftrace_ops *op;
2181	unsigned long ip = rec->ip;
2182
2183	do_for_each_ftrace_op(op, ftrace_ops_list) {
2184
2185		if (!op->trampoline)
2186			continue;
2187
2188		if (hash_contains_ip(ip, op->func_hash))
2189			return op;
2190	} while_for_each_ftrace_op(op);
2191
2192	return NULL;
2193}
2194
2195static struct ftrace_ops *
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2196ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
2197			   struct ftrace_ops *op)
2198{
2199	unsigned long ip = rec->ip;
2200
2201	while_for_each_ftrace_op(op) {
2202
2203		if (!op->trampoline)
2204			continue;
2205
2206		if (hash_contains_ip(ip, op->func_hash))
2207			return op;
2208	} 
2209
2210	return NULL;
2211}
2212
2213static struct ftrace_ops *
2214ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
2215{
2216	struct ftrace_ops *op;
2217	unsigned long ip = rec->ip;
2218
2219	/*
2220	 * Need to check removed ops first.
2221	 * If they are being removed, and this rec has a tramp,
2222	 * and this rec is in the ops list, then it would be the
2223	 * one with the tramp.
2224	 */
2225	if (removed_ops) {
2226		if (hash_contains_ip(ip, &removed_ops->old_hash))
2227			return removed_ops;
2228	}
2229
2230	/*
2231	 * Need to find the current trampoline for a rec.
2232	 * Now, a trampoline is only attached to a rec if there
2233	 * was a single 'ops' attached to it. But this can be called
2234	 * when we are adding another op to the rec or removing the
2235	 * current one. Thus, if the op is being added, we can
2236	 * ignore it because it hasn't attached itself to the rec
2237	 * yet.
2238	 *
2239	 * If an ops is being modified (hooking to different functions)
2240	 * then we don't care about the new functions that are being
2241	 * added, just the old ones (that are probably being removed).
2242	 *
2243	 * If we are adding an ops to a function that already is using
2244	 * a trampoline, it needs to be removed (trampolines are only
2245	 * for single ops connected), then an ops that is not being
2246	 * modified also needs to be checked.
2247	 */
2248	do_for_each_ftrace_op(op, ftrace_ops_list) {
2249
2250		if (!op->trampoline)
2251			continue;
2252
2253		/*
2254		 * If the ops is being added, it hasn't gotten to
2255		 * the point to be removed from this tree yet.
2256		 */
2257		if (op->flags & FTRACE_OPS_FL_ADDING)
2258			continue;
2259
2260
2261		/*
2262		 * If the ops is being modified and is in the old
2263		 * hash, then it is probably being removed from this
2264		 * function.
2265		 */
2266		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
2267		    hash_contains_ip(ip, &op->old_hash))
2268			return op;
2269		/*
2270		 * If the ops is not being added or modified, and it's
2271		 * in its normal filter hash, then this must be the one
2272		 * we want!
2273		 */
2274		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
2275		    hash_contains_ip(ip, op->func_hash))
2276			return op;
2277
2278	} while_for_each_ftrace_op(op);
2279
2280	return NULL;
2281}
2282
2283static struct ftrace_ops *
2284ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
2285{
2286	struct ftrace_ops *op;
2287	unsigned long ip = rec->ip;
2288
2289	do_for_each_ftrace_op(op, ftrace_ops_list) {
2290		/* pass rec in as regs to have non-NULL val */
2291		if (hash_contains_ip(ip, op->func_hash))
2292			return op;
2293	} while_for_each_ftrace_op(op);
2294
2295	return NULL;
2296}
2297
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2298/**
2299 * ftrace_get_addr_new - Get the call address to set to
2300 * @rec:  The ftrace record descriptor
2301 *
2302 * If the record has the FTRACE_FL_REGS set, that means that it
2303 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
2304 * is not not set, then it wants to convert to the normal callback.
2305 *
2306 * Returns the address of the trampoline to set to
2307 */
2308unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
2309{
2310	struct ftrace_ops *ops;
 
 
 
 
 
 
 
 
 
2311
2312	/* Trampolines take precedence over regs */
2313	if (rec->flags & FTRACE_FL_TRAMP) {
2314		ops = ftrace_find_tramp_ops_new(rec);
2315		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2316			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
2317				(void *)rec->ip, (void *)rec->ip, rec->flags);
2318			/* Ftrace is shutting down, return anything */
2319			return (unsigned long)FTRACE_ADDR;
2320		}
2321		return ops->trampoline;
2322	}
2323
2324	if (rec->flags & FTRACE_FL_REGS)
2325		return (unsigned long)FTRACE_REGS_ADDR;
2326	else
2327		return (unsigned long)FTRACE_ADDR;
2328}
2329
2330/**
2331 * ftrace_get_addr_curr - Get the call address that is already there
2332 * @rec:  The ftrace record descriptor
2333 *
2334 * The FTRACE_FL_REGS_EN is set when the record already points to
2335 * a function that saves all the regs. Basically the '_EN' version
2336 * represents the current state of the function.
2337 *
2338 * Returns the address of the trampoline that is currently being called
2339 */
2340unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
2341{
2342	struct ftrace_ops *ops;
 
 
 
 
 
 
 
 
 
2343
2344	/* Trampolines take precedence over regs */
2345	if (rec->flags & FTRACE_FL_TRAMP_EN) {
2346		ops = ftrace_find_tramp_ops_curr(rec);
2347		if (FTRACE_WARN_ON(!ops)) {
2348			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
2349				(void *)rec->ip, (void *)rec->ip);
2350			/* Ftrace is shutting down, return anything */
2351			return (unsigned long)FTRACE_ADDR;
2352		}
2353		return ops->trampoline;
2354	}
2355
2356	if (rec->flags & FTRACE_FL_REGS_EN)
2357		return (unsigned long)FTRACE_REGS_ADDR;
2358	else
2359		return (unsigned long)FTRACE_ADDR;
2360}
2361
2362static int
2363__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
2364{
2365	unsigned long ftrace_old_addr;
2366	unsigned long ftrace_addr;
2367	int ret;
2368
2369	ftrace_addr = ftrace_get_addr_new(rec);
2370
2371	/* This needs to be done before we call ftrace_update_record */
2372	ftrace_old_addr = ftrace_get_addr_curr(rec);
2373
2374	ret = ftrace_update_record(rec, enable);
2375
2376	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2377
2378	switch (ret) {
2379	case FTRACE_UPDATE_IGNORE:
2380		return 0;
2381
2382	case FTRACE_UPDATE_MAKE_CALL:
2383		ftrace_bug_type = FTRACE_BUG_CALL;
2384		return ftrace_make_call(rec, ftrace_addr);
2385
2386	case FTRACE_UPDATE_MAKE_NOP:
2387		ftrace_bug_type = FTRACE_BUG_NOP;
2388		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2389
2390	case FTRACE_UPDATE_MODIFY_CALL:
2391		ftrace_bug_type = FTRACE_BUG_UPDATE;
2392		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2393	}
2394
2395	return -1; /* unknow ftrace bug */
2396}
2397
2398void __weak ftrace_replace_code(int enable)
2399{
2400	struct dyn_ftrace *rec;
2401	struct ftrace_page *pg;
 
 
2402	int failed;
2403
2404	if (unlikely(ftrace_disabled))
2405		return;
2406
2407	do_for_each_ftrace_rec(pg, rec) {
2408
2409		if (rec->flags & FTRACE_FL_DISABLED)
2410			continue;
2411
2412		failed = __ftrace_replace_code(rec, enable);
2413		if (failed) {
2414			ftrace_bug(failed, rec);
2415			/* Stop processing */
2416			return;
2417		}
 
 
2418	} while_for_each_ftrace_rec();
2419}
2420
2421struct ftrace_rec_iter {
2422	struct ftrace_page	*pg;
2423	int			index;
2424};
2425
2426/**
2427 * ftrace_rec_iter_start, start up iterating over traced functions
2428 *
2429 * Returns an iterator handle that is used to iterate over all
2430 * the records that represent address locations where functions
2431 * are traced.
2432 *
2433 * May return NULL if no records are available.
2434 */
2435struct ftrace_rec_iter *ftrace_rec_iter_start(void)
2436{
2437	/*
2438	 * We only use a single iterator.
2439	 * Protected by the ftrace_lock mutex.
2440	 */
2441	static struct ftrace_rec_iter ftrace_rec_iter;
2442	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
2443
2444	iter->pg = ftrace_pages_start;
2445	iter->index = 0;
2446
2447	/* Could have empty pages */
2448	while (iter->pg && !iter->pg->index)
2449		iter->pg = iter->pg->next;
2450
2451	if (!iter->pg)
2452		return NULL;
2453
2454	return iter;
2455}
2456
2457/**
2458 * ftrace_rec_iter_next, get the next record to process.
2459 * @iter: The handle to the iterator.
2460 *
2461 * Returns the next iterator after the given iterator @iter.
2462 */
2463struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
2464{
2465	iter->index++;
2466
2467	if (iter->index >= iter->pg->index) {
2468		iter->pg = iter->pg->next;
2469		iter->index = 0;
2470
2471		/* Could have empty pages */
2472		while (iter->pg && !iter->pg->index)
2473			iter->pg = iter->pg->next;
2474	}
2475
2476	if (!iter->pg)
2477		return NULL;
2478
2479	return iter;
2480}
2481
2482/**
2483 * ftrace_rec_iter_record, get the record at the iterator location
2484 * @iter: The current iterator location
2485 *
2486 * Returns the record that the current @iter is at.
2487 */
2488struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
2489{
2490	return &iter->pg->records[iter->index];
2491}
2492
2493static int
2494ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
2495{
2496	int ret;
2497
2498	if (unlikely(ftrace_disabled))
2499		return 0;
2500
2501	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
2502	if (ret) {
2503		ftrace_bug_type = FTRACE_BUG_INIT;
2504		ftrace_bug(ret, rec);
2505		return 0;
2506	}
2507	return 1;
2508}
2509
2510/*
2511 * archs can override this function if they must do something
2512 * before the modifying code is performed.
2513 */
2514int __weak ftrace_arch_code_modify_prepare(void)
2515{
2516	return 0;
2517}
2518
2519/*
2520 * archs can override this function if they must do something
2521 * after the modifying code is performed.
2522 */
2523int __weak ftrace_arch_code_modify_post_process(void)
2524{
2525	return 0;
2526}
2527
2528void ftrace_modify_all_code(int command)
2529{
2530	int update = command & FTRACE_UPDATE_TRACE_FUNC;
 
2531	int err = 0;
2532
 
 
 
2533	/*
2534	 * If the ftrace_caller calls a ftrace_ops func directly,
2535	 * we need to make sure that it only traces functions it
2536	 * expects to trace. When doing the switch of functions,
2537	 * we need to update to the ftrace_ops_list_func first
2538	 * before the transition between old and new calls are set,
2539	 * as the ftrace_ops_list_func will check the ops hashes
2540	 * to make sure the ops are having the right functions
2541	 * traced.
2542	 */
2543	if (update) {
2544		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
2545		if (FTRACE_WARN_ON(err))
2546			return;
2547	}
2548
2549	if (command & FTRACE_UPDATE_CALLS)
2550		ftrace_replace_code(1);
2551	else if (command & FTRACE_DISABLE_CALLS)
2552		ftrace_replace_code(0);
2553
2554	if (update && ftrace_trace_function != ftrace_ops_list_func) {
2555		function_trace_op = set_function_trace_op;
2556		smp_wmb();
2557		/* If irqs are disabled, we are in stop machine */
2558		if (!irqs_disabled())
2559			smp_call_function(ftrace_sync_ipi, NULL, 1);
2560		err = ftrace_update_ftrace_func(ftrace_trace_function);
2561		if (FTRACE_WARN_ON(err))
2562			return;
2563	}
2564
2565	if (command & FTRACE_START_FUNC_RET)
2566		err = ftrace_enable_ftrace_graph_caller();
2567	else if (command & FTRACE_STOP_FUNC_RET)
2568		err = ftrace_disable_ftrace_graph_caller();
2569	FTRACE_WARN_ON(err);
2570}
2571
2572static int __ftrace_modify_code(void *data)
2573{
2574	int *command = data;
2575
2576	ftrace_modify_all_code(*command);
2577
2578	return 0;
2579}
2580
2581/**
2582 * ftrace_run_stop_machine, go back to the stop machine method
2583 * @command: The command to tell ftrace what to do
2584 *
2585 * If an arch needs to fall back to the stop machine method, the
2586 * it can call this function.
2587 */
2588void ftrace_run_stop_machine(int command)
2589{
2590	stop_machine(__ftrace_modify_code, &command, NULL);
2591}
2592
2593/**
2594 * arch_ftrace_update_code, modify the code to trace or not trace
2595 * @command: The command that needs to be done
2596 *
2597 * Archs can override this function if it does not need to
2598 * run stop_machine() to modify code.
2599 */
2600void __weak arch_ftrace_update_code(int command)
2601{
2602	ftrace_run_stop_machine(command);
2603}
2604
2605static void ftrace_run_update_code(int command)
2606{
2607	int ret;
2608
2609	ret = ftrace_arch_code_modify_prepare();
2610	FTRACE_WARN_ON(ret);
2611	if (ret)
2612		return;
2613
2614	/*
2615	 * By default we use stop_machine() to modify the code.
2616	 * But archs can do what ever they want as long as it
2617	 * is safe. The stop_machine() is the safest, but also
2618	 * produces the most overhead.
2619	 */
2620	arch_ftrace_update_code(command);
2621
2622	ret = ftrace_arch_code_modify_post_process();
2623	FTRACE_WARN_ON(ret);
2624}
2625
2626static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2627				   struct ftrace_ops_hash *old_hash)
2628{
2629	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2630	ops->old_hash.filter_hash = old_hash->filter_hash;
2631	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2632	ftrace_run_update_code(command);
2633	ops->old_hash.filter_hash = NULL;
2634	ops->old_hash.notrace_hash = NULL;
2635	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
2636}
2637
2638static ftrace_func_t saved_ftrace_func;
2639static int ftrace_start_up;
2640
2641void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
2642{
2643}
2644
2645static void per_cpu_ops_free(struct ftrace_ops *ops)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2646{
2647	free_percpu(ops->disabled);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2648}
2649
2650static void ftrace_startup_enable(int command)
2651{
2652	if (saved_ftrace_func != ftrace_trace_function) {
2653		saved_ftrace_func = ftrace_trace_function;
2654		command |= FTRACE_UPDATE_TRACE_FUNC;
2655	}
2656
2657	if (!command || !ftrace_enabled)
2658		return;
2659
2660	ftrace_run_update_code(command);
2661}
2662
2663static void ftrace_startup_all(int command)
2664{
2665	update_all_ops = true;
2666	ftrace_startup_enable(command);
2667	update_all_ops = false;
2668}
2669
2670static int ftrace_startup(struct ftrace_ops *ops, int command)
2671{
2672	int ret;
2673
2674	if (unlikely(ftrace_disabled))
2675		return -ENODEV;
2676
2677	ret = __register_ftrace_function(ops);
2678	if (ret)
2679		return ret;
2680
2681	ftrace_start_up++;
2682
2683	/*
2684	 * Note that ftrace probes uses this to start up
2685	 * and modify functions it will probe. But we still
2686	 * set the ADDING flag for modification, as probes
2687	 * do not have trampolines. If they add them in the
2688	 * future, then the probes will need to distinguish
2689	 * between adding and updating probes.
2690	 */
2691	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2692
2693	ret = ftrace_hash_ipmodify_enable(ops);
2694	if (ret < 0) {
2695		/* Rollback registration process */
2696		__unregister_ftrace_function(ops);
2697		ftrace_start_up--;
2698		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 
 
2699		return ret;
2700	}
2701
2702	if (ftrace_hash_rec_enable(ops, 1))
2703		command |= FTRACE_UPDATE_CALLS;
2704
2705	ftrace_startup_enable(command);
2706
2707	ops->flags &= ~FTRACE_OPS_FL_ADDING;
2708
2709	return 0;
2710}
2711
2712static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2713{
2714	int ret;
2715
2716	if (unlikely(ftrace_disabled))
2717		return -ENODEV;
2718
2719	ret = __unregister_ftrace_function(ops);
2720	if (ret)
2721		return ret;
2722
2723	ftrace_start_up--;
2724	/*
2725	 * Just warn in case of unbalance, no need to kill ftrace, it's not
2726	 * critical but the ftrace_call callers may be never nopped again after
2727	 * further ftrace uses.
2728	 */
2729	WARN_ON_ONCE(ftrace_start_up < 0);
2730
2731	/* Disabling ipmodify never fails */
2732	ftrace_hash_ipmodify_disable(ops);
2733
2734	if (ftrace_hash_rec_disable(ops, 1))
2735		command |= FTRACE_UPDATE_CALLS;
2736
2737	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2738
2739	if (saved_ftrace_func != ftrace_trace_function) {
2740		saved_ftrace_func = ftrace_trace_function;
2741		command |= FTRACE_UPDATE_TRACE_FUNC;
2742	}
2743
2744	if (!command || !ftrace_enabled) {
2745		/*
2746		 * If these are per_cpu ops, they still need their
2747		 * per_cpu field freed. Since, function tracing is
2748		 * not currently active, we can just free them
2749		 * without synchronizing all CPUs.
2750		 */
2751		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
2752			per_cpu_ops_free(ops);
 
2753		return 0;
2754	}
2755
2756	/*
2757	 * If the ops uses a trampoline, then it needs to be
2758	 * tested first on update.
2759	 */
2760	ops->flags |= FTRACE_OPS_FL_REMOVING;
2761	removed_ops = ops;
2762
2763	/* The trampoline logic checks the old hashes */
2764	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
2765	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
2766
2767	ftrace_run_update_code(command);
2768
2769	/*
2770	 * If there's no more ops registered with ftrace, run a
2771	 * sanity check to make sure all rec flags are cleared.
2772	 */
2773	if (ftrace_ops_list == &ftrace_list_end) {
 
2774		struct ftrace_page *pg;
2775		struct dyn_ftrace *rec;
2776
2777		do_for_each_ftrace_rec(pg, rec) {
2778			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
2779				pr_warn("  %pS flags:%lx\n",
2780					(void *)rec->ip, rec->flags);
2781		} while_for_each_ftrace_rec();
2782	}
2783
2784	ops->old_hash.filter_hash = NULL;
2785	ops->old_hash.notrace_hash = NULL;
2786
2787	removed_ops = NULL;
2788	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2789
2790	/*
2791	 * Dynamic ops may be freed, we must make sure that all
2792	 * callers are done before leaving this function.
2793	 * The same goes for freeing the per_cpu data of the per_cpu
2794	 * ops.
2795	 *
2796	 * Again, normal synchronize_sched() is not good enough.
2797	 * We need to do a hard force of sched synchronization.
2798	 * This is because we use preempt_disable() to do RCU, but
2799	 * the function tracers can be called where RCU is not watching
2800	 * (like before user_exit()). We can not rely on the RCU
2801	 * infrastructure to do the synchronization, thus we must do it
2802	 * ourselves.
2803	 */
2804	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) {
2805		schedule_on_each_cpu(ftrace_sync);
 
 
 
 
 
 
 
 
2806
2807		arch_ftrace_trampoline_free(ops);
 
 
 
 
 
 
 
 
2808
2809		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
2810			per_cpu_ops_free(ops);
2811	}
2812
2813	return 0;
2814}
2815
2816static void ftrace_startup_sysctl(void)
2817{
2818	int command;
2819
2820	if (unlikely(ftrace_disabled))
2821		return;
2822
2823	/* Force update next time */
2824	saved_ftrace_func = NULL;
2825	/* ftrace_start_up is true if we want ftrace running */
2826	if (ftrace_start_up) {
2827		command = FTRACE_UPDATE_CALLS;
2828		if (ftrace_graph_active)
2829			command |= FTRACE_START_FUNC_RET;
2830		ftrace_startup_enable(command);
2831	}
2832}
2833
2834static void ftrace_shutdown_sysctl(void)
2835{
2836	int command;
2837
2838	if (unlikely(ftrace_disabled))
2839		return;
2840
2841	/* ftrace_start_up is true if ftrace is running */
2842	if (ftrace_start_up) {
2843		command = FTRACE_DISABLE_CALLS;
2844		if (ftrace_graph_active)
2845			command |= FTRACE_STOP_FUNC_RET;
2846		ftrace_run_update_code(command);
2847	}
2848}
2849
2850static u64		ftrace_update_time;
2851unsigned long		ftrace_update_tot_cnt;
 
 
2852
2853static inline int ops_traces_mod(struct ftrace_ops *ops)
2854{
2855	/*
2856	 * Filter_hash being empty will default to trace module.
2857	 * But notrace hash requires a test of individual module functions.
2858	 */
2859	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
2860		ftrace_hash_empty(ops->func_hash->notrace_hash);
2861}
2862
2863/*
2864 * Check if the current ops references the record.
2865 *
2866 * If the ops traces all functions, then it was already accounted for.
2867 * If the ops does not trace the current record function, skip it.
2868 * If the ops ignores the function via notrace filter, skip it.
2869 */
2870static inline bool
2871ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
2872{
2873	/* If ops isn't enabled, ignore it */
2874	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
2875		return 0;
2876
2877	/* If ops traces all then it includes this function */
2878	if (ops_traces_mod(ops))
2879		return 1;
2880
2881	/* The function must be in the filter */
2882	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2883	    !ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2884		return 0;
2885
2886	/* If in notrace hash, we ignore it too */
2887	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2888		return 0;
2889
2890	return 1;
2891}
2892
2893static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2894{
2895	struct ftrace_page *pg;
2896	struct dyn_ftrace *p;
2897	u64 start, stop;
2898	unsigned long update_cnt = 0;
2899	unsigned long rec_flags = 0;
2900	int i;
2901
2902	start = ftrace_now(raw_smp_processor_id());
2903
2904	/*
2905	 * When a module is loaded, this function is called to convert
2906	 * the calls to mcount in its text to nops, and also to create
2907	 * an entry in the ftrace data. Now, if ftrace is activated
2908	 * after this call, but before the module sets its text to
2909	 * read-only, the modification of enabling ftrace can fail if
2910	 * the read-only is done while ftrace is converting the calls.
2911	 * To prevent this, the module's records are set as disabled
2912	 * and will be enabled after the call to set the module's text
2913	 * to read-only.
2914	 */
2915	if (mod)
2916		rec_flags |= FTRACE_FL_DISABLED;
2917
2918	for (pg = new_pgs; pg; pg = pg->next) {
2919
2920		for (i = 0; i < pg->index; i++) {
2921
2922			/* If something went wrong, bail without enabling anything */
2923			if (unlikely(ftrace_disabled))
2924				return -1;
2925
2926			p = &pg->records[i];
2927			p->flags = rec_flags;
2928
2929			/*
2930			 * Do the initial record conversion from mcount jump
2931			 * to the NOP instructions.
2932			 */
2933			if (!ftrace_code_disable(mod, p))
 
2934				break;
2935
2936			update_cnt++;
2937		}
2938	}
2939
2940	stop = ftrace_now(raw_smp_processor_id());
2941	ftrace_update_time = stop - start;
2942	ftrace_update_tot_cnt += update_cnt;
2943
2944	return 0;
2945}
2946
2947static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2948{
2949	int order;
 
2950	int cnt;
2951
2952	if (WARN_ON(!count))
2953		return -EINVAL;
2954
2955	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2956
2957	/*
2958	 * We want to fill as much as possible. No more than a page
2959	 * may be empty.
2960	 */
2961	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
2962		order--;
2963
2964 again:
2965	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2966
2967	if (!pg->records) {
2968		/* if we can't allocate this size, try something smaller */
2969		if (!order)
2970			return -ENOMEM;
2971		order >>= 1;
2972		goto again;
2973	}
2974
 
 
 
2975	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
2976	pg->size = cnt;
2977
2978	if (cnt > count)
2979		cnt = count;
2980
2981	return cnt;
2982}
2983
2984static struct ftrace_page *
2985ftrace_allocate_pages(unsigned long num_to_init)
2986{
2987	struct ftrace_page *start_pg;
2988	struct ftrace_page *pg;
2989	int order;
2990	int cnt;
2991
2992	if (!num_to_init)
2993		return 0;
2994
2995	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
2996	if (!pg)
2997		return NULL;
2998
2999	/*
3000	 * Try to allocate as much as possible in one continues
3001	 * location that fills in all of the space. We want to
3002	 * waste as little space as possible.
3003	 */
3004	for (;;) {
3005		cnt = ftrace_allocate_records(pg, num_to_init);
3006		if (cnt < 0)
3007			goto free_pages;
3008
3009		num_to_init -= cnt;
3010		if (!num_to_init)
3011			break;
3012
3013		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
3014		if (!pg->next)
3015			goto free_pages;
3016
3017		pg = pg->next;
3018	}
3019
3020	return start_pg;
3021
3022 free_pages:
3023	pg = start_pg;
3024	while (pg) {
3025		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
3026		free_pages((unsigned long)pg->records, order);
 
 
3027		start_pg = pg->next;
3028		kfree(pg);
3029		pg = start_pg;
 
3030	}
3031	pr_info("ftrace: FAILED to allocate memory for functions\n");
3032	return NULL;
3033}
3034
3035#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
3036
3037struct ftrace_iterator {
3038	loff_t				pos;
3039	loff_t				func_pos;
 
3040	struct ftrace_page		*pg;
3041	struct dyn_ftrace		*func;
3042	struct ftrace_func_probe	*probe;
 
3043	struct trace_parser		parser;
3044	struct ftrace_hash		*hash;
3045	struct ftrace_ops		*ops;
3046	int				hidx;
 
 
3047	int				idx;
3048	unsigned			flags;
3049};
3050
3051static void *
3052t_hash_next(struct seq_file *m, loff_t *pos)
3053{
3054	struct ftrace_iterator *iter = m->private;
 
 
 
 
3055	struct hlist_node *hnd = NULL;
3056	struct hlist_head *hhd;
 
3057
3058	(*pos)++;
3059	iter->pos = *pos;
3060
3061	if (iter->probe)
3062		hnd = &iter->probe->node;
3063 retry:
3064	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3065		return NULL;
3066
3067	hhd = &ftrace_func_hash[iter->hidx];
 
 
 
 
 
 
 
 
 
 
 
 
 
3068
3069	if (hlist_empty(hhd)) {
3070		iter->hidx++;
3071		hnd = NULL;
3072		goto retry;
3073	}
3074
3075	if (!hnd)
3076		hnd = hhd->first;
3077	else {
3078		hnd = hnd->next;
3079		if (!hnd) {
3080			iter->hidx++;
3081			goto retry;
3082		}
3083	}
3084
3085	if (WARN_ON_ONCE(!hnd))
3086		return NULL;
3087
3088	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
3089
3090	return iter;
3091}
3092
3093static void *t_hash_start(struct seq_file *m, loff_t *pos)
3094{
3095	struct ftrace_iterator *iter = m->private;
3096	void *p = NULL;
3097	loff_t l;
3098
3099	if (!(iter->flags & FTRACE_ITER_DO_HASH))
3100		return NULL;
3101
3102	if (iter->func_pos > *pos)
3103		return NULL;
3104
3105	iter->hidx = 0;
3106	for (l = 0; l <= (*pos - iter->func_pos); ) {
3107		p = t_hash_next(m, &l);
 
 
3108		if (!p)
3109			break;
3110	}
3111	if (!p)
3112		return NULL;
3113
3114	/* Only set this if we have an item */
3115	iter->flags |= FTRACE_ITER_HASH;
3116
3117	return iter;
3118}
3119
3120static int
3121t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
3122{
3123	struct ftrace_func_probe *rec;
 
 
3124
3125	rec = iter->probe;
3126	if (WARN_ON_ONCE(!rec))
 
 
3127		return -EIO;
3128
3129	if (rec->ops->print)
3130		return rec->ops->print(m, rec->ip, rec->ops, rec->data);
3131
3132	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
 
3133
3134	if (rec->data)
3135		seq_printf(m, ":%p", rec->data);
3136	seq_putc(m, '\n');
3137
3138	return 0;
3139}
3140
3141static void *
3142t_next(struct seq_file *m, void *v, loff_t *pos)
3143{
3144	struct ftrace_iterator *iter = m->private;
3145	struct ftrace_ops *ops = iter->ops;
3146	struct dyn_ftrace *rec = NULL;
3147
3148	if (unlikely(ftrace_disabled))
 
 
 
 
 
 
 
3149		return NULL;
 
3150
3151	if (iter->flags & FTRACE_ITER_HASH)
3152		return t_hash_next(m, pos);
3153
3154	(*pos)++;
3155	iter->pos = iter->func_pos = *pos;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3156
3157	if (iter->flags & FTRACE_ITER_PRINTALL)
3158		return t_hash_start(m, pos);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3159
3160 retry:
3161	if (iter->idx >= iter->pg->index) {
3162		if (iter->pg->next) {
3163			iter->pg = iter->pg->next;
3164			iter->idx = 0;
3165			goto retry;
3166		}
3167	} else {
3168		rec = &iter->pg->records[iter->idx++];
3169		if (((iter->flags & FTRACE_ITER_FILTER) &&
3170		     !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) ||
3171
3172		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
3173		     !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) ||
3174
3175		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3176		     !(rec->flags & FTRACE_FL_ENABLED))) {
3177
3178			rec = NULL;
3179			goto retry;
3180		}
3181	}
3182
3183	if (!rec)
3184		return t_hash_start(m, pos);
3185
 
3186	iter->func = rec;
3187
3188	return iter;
3189}
3190
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3191static void reset_iter_read(struct ftrace_iterator *iter)
3192{
3193	iter->pos = 0;
3194	iter->func_pos = 0;
3195	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
3196}
3197
3198static void *t_start(struct seq_file *m, loff_t *pos)
3199{
3200	struct ftrace_iterator *iter = m->private;
3201	struct ftrace_ops *ops = iter->ops;
3202	void *p = NULL;
3203	loff_t l;
3204
3205	mutex_lock(&ftrace_lock);
3206
3207	if (unlikely(ftrace_disabled))
3208		return NULL;
3209
3210	/*
3211	 * If an lseek was done, then reset and start from beginning.
3212	 */
3213	if (*pos < iter->pos)
3214		reset_iter_read(iter);
3215
3216	/*
3217	 * For set_ftrace_filter reading, if we have the filter
3218	 * off, we can short cut and just print out that all
3219	 * functions are enabled.
3220	 */
3221	if ((iter->flags & FTRACE_ITER_FILTER &&
3222	     ftrace_hash_empty(ops->func_hash->filter_hash)) ||
3223	    (iter->flags & FTRACE_ITER_NOTRACE &&
3224	     ftrace_hash_empty(ops->func_hash->notrace_hash))) {
3225		if (*pos > 0)
3226			return t_hash_start(m, pos);
3227		iter->flags |= FTRACE_ITER_PRINTALL;
3228		/* reset in case of seek/pread */
3229		iter->flags &= ~FTRACE_ITER_HASH;
3230		return iter;
3231	}
3232
3233	if (iter->flags & FTRACE_ITER_HASH)
3234		return t_hash_start(m, pos);
3235
3236	/*
3237	 * Unfortunately, we need to restart at ftrace_pages_start
3238	 * every time we let go of the ftrace_mutex. This is because
3239	 * those pointers can change without the lock.
3240	 */
3241	iter->pg = ftrace_pages_start;
3242	iter->idx = 0;
3243	for (l = 0; l <= *pos; ) {
3244		p = t_next(m, p, &l);
3245		if (!p)
3246			break;
3247	}
3248
3249	if (!p)
3250		return t_hash_start(m, pos);
3251
3252	return iter;
3253}
3254
3255static void t_stop(struct seq_file *m, void *p)
3256{
3257	mutex_unlock(&ftrace_lock);
3258}
3259
3260void * __weak
3261arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
3262{
3263	return NULL;
3264}
3265
3266static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
3267				struct dyn_ftrace *rec)
3268{
3269	void *ptr;
3270
3271	ptr = arch_ftrace_trampoline_func(ops, rec);
3272	if (ptr)
3273		seq_printf(m, " ->%pS", ptr);
3274}
3275
3276static int t_show(struct seq_file *m, void *v)
3277{
3278	struct ftrace_iterator *iter = m->private;
3279	struct dyn_ftrace *rec;
3280
3281	if (iter->flags & FTRACE_ITER_HASH)
3282		return t_hash_show(m, iter);
 
 
 
3283
3284	if (iter->flags & FTRACE_ITER_PRINTALL) {
3285		if (iter->flags & FTRACE_ITER_NOTRACE)
3286			seq_puts(m, "#### no functions disabled ####\n");
3287		else
3288			seq_puts(m, "#### all functions enabled ####\n");
3289		return 0;
3290	}
3291
3292	rec = iter->func;
3293
3294	if (!rec)
3295		return 0;
3296
3297	seq_printf(m, "%ps", (void *)rec->ip);
3298	if (iter->flags & FTRACE_ITER_ENABLED) {
3299		struct ftrace_ops *ops;
3300
3301		seq_printf(m, " (%ld)%s%s",
3302			   ftrace_rec_count(rec),
3303			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
3304			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
 
3305		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3306			ops = ftrace_find_tramp_ops_any(rec);
3307			if (ops) {
3308				do {
3309					seq_printf(m, "\ttramp: %pS (%pS)",
3310						   (void *)ops->trampoline,
3311						   (void *)ops->func);
3312					add_trampoline_func(m, ops, rec);
3313					ops = ftrace_find_tramp_ops_next(rec, ops);
3314				} while (ops);
3315			} else
3316				seq_puts(m, "\ttramp: ERROR!");
3317		} else {
3318			add_trampoline_func(m, NULL, rec);
3319		}
3320	}	
 
 
 
 
 
 
 
3321
3322	seq_putc(m, '\n');
3323
3324	return 0;
3325}
3326
3327static const struct seq_operations show_ftrace_seq_ops = {
3328	.start = t_start,
3329	.next = t_next,
3330	.stop = t_stop,
3331	.show = t_show,
3332};
3333
3334static int
3335ftrace_avail_open(struct inode *inode, struct file *file)
3336{
3337	struct ftrace_iterator *iter;
 
 
 
 
 
3338
3339	if (unlikely(ftrace_disabled))
3340		return -ENODEV;
3341
3342	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3343	if (iter) {
3344		iter->pg = ftrace_pages_start;
3345		iter->ops = &global_ops;
3346	}
 
3347
3348	return iter ? 0 : -ENOMEM;
3349}
3350
3351static int
3352ftrace_enabled_open(struct inode *inode, struct file *file)
3353{
3354	struct ftrace_iterator *iter;
3355
 
 
 
 
 
 
 
 
 
3356	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3357	if (iter) {
3358		iter->pg = ftrace_pages_start;
3359		iter->flags = FTRACE_ITER_ENABLED;
3360		iter->ops = &global_ops;
3361	}
3362
3363	return iter ? 0 : -ENOMEM;
 
 
 
 
3364}
3365
3366/**
3367 * ftrace_regex_open - initialize function tracer filter files
3368 * @ops: The ftrace_ops that hold the hash filters
3369 * @flag: The type of filter to process
3370 * @inode: The inode, usually passed in to your open routine
3371 * @file: The file, usually passed in to your open routine
3372 *
3373 * ftrace_regex_open() initializes the filter files for the
3374 * @ops. Depending on @flag it may process the filter hash or
3375 * the notrace hash of @ops. With this called from the open
3376 * routine, you can use ftrace_filter_write() for the write
3377 * routine if @flag has FTRACE_ITER_FILTER set, or
3378 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3379 * tracing_lseek() should be used as the lseek routine, and
3380 * release must call ftrace_regex_release().
3381 */
3382int
3383ftrace_regex_open(struct ftrace_ops *ops, int flag,
3384		  struct inode *inode, struct file *file)
3385{
3386	struct ftrace_iterator *iter;
3387	struct ftrace_hash *hash;
3388	int ret = 0;
 
 
3389
3390	ftrace_ops_init(ops);
3391
3392	if (unlikely(ftrace_disabled))
3393		return -ENODEV;
3394
 
 
 
3395	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3396	if (!iter)
3397		return -ENOMEM;
3398
3399	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
3400		kfree(iter);
3401		return -ENOMEM;
3402	}
3403
3404	iter->ops = ops;
3405	iter->flags = flag;
 
3406
3407	mutex_lock(&ops->func_hash->regex_lock);
3408
3409	if (flag & FTRACE_ITER_NOTRACE)
3410		hash = ops->func_hash->notrace_hash;
3411	else
 
3412		hash = ops->func_hash->filter_hash;
 
 
 
 
3413
3414	if (file->f_mode & FMODE_WRITE) {
3415		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
3416
3417		if (file->f_flags & O_TRUNC)
3418			iter->hash = alloc_ftrace_hash(size_bits);
3419		else
 
3420			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
 
3421
3422		if (!iter->hash) {
3423			trace_parser_put(&iter->parser);
3424			kfree(iter);
3425			ret = -ENOMEM;
3426			goto out_unlock;
3427		}
3428	}
 
 
 
3429
3430	if (file->f_mode & FMODE_READ) {
3431		iter->pg = ftrace_pages_start;
3432
3433		ret = seq_open(file, &show_ftrace_seq_ops);
3434		if (!ret) {
3435			struct seq_file *m = file->private_data;
3436			m->private = iter;
3437		} else {
3438			/* Failed */
3439			free_ftrace_hash(iter->hash);
3440			trace_parser_put(&iter->parser);
3441			kfree(iter);
3442		}
3443	} else
3444		file->private_data = iter;
3445
3446 out_unlock:
3447	mutex_unlock(&ops->func_hash->regex_lock);
3448
 
 
 
 
 
 
 
3449	return ret;
3450}
3451
3452static int
3453ftrace_filter_open(struct inode *inode, struct file *file)
3454{
3455	struct ftrace_ops *ops = inode->i_private;
3456
 
3457	return ftrace_regex_open(ops,
3458			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
3459			inode, file);
3460}
3461
3462static int
3463ftrace_notrace_open(struct inode *inode, struct file *file)
3464{
3465	struct ftrace_ops *ops = inode->i_private;
3466
 
3467	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3468				 inode, file);
3469}
3470
3471/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
3472struct ftrace_glob {
3473	char *search;
3474	unsigned len;
3475	int type;
3476};
3477
3478/*
3479 * If symbols in an architecture don't correspond exactly to the user-visible
3480 * name of what they represent, it is possible to define this function to
3481 * perform the necessary adjustments.
3482*/
3483char * __weak arch_ftrace_match_adjust(char *str, const char *search)
3484{
3485	return str;
3486}
3487
3488static int ftrace_match(char *str, struct ftrace_glob *g)
3489{
3490	int matched = 0;
3491	int slen;
3492
3493	str = arch_ftrace_match_adjust(str, g->search);
3494
3495	switch (g->type) {
3496	case MATCH_FULL:
3497		if (strcmp(str, g->search) == 0)
3498			matched = 1;
3499		break;
3500	case MATCH_FRONT_ONLY:
3501		if (strncmp(str, g->search, g->len) == 0)
3502			matched = 1;
3503		break;
3504	case MATCH_MIDDLE_ONLY:
3505		if (strstr(str, g->search))
3506			matched = 1;
3507		break;
3508	case MATCH_END_ONLY:
3509		slen = strlen(str);
3510		if (slen >= g->len &&
3511		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3512			matched = 1;
3513		break;
3514	case MATCH_GLOB:
3515		if (glob_match(g->search, str))
3516			matched = 1;
3517		break;
3518	}
3519
3520	return matched;
3521}
3522
3523static int
3524enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3525{
3526	struct ftrace_func_entry *entry;
3527	int ret = 0;
3528
3529	entry = ftrace_lookup_ip(hash, rec->ip);
3530	if (clear_filter) {
3531		/* Do nothing if it doesn't exist */
3532		if (!entry)
3533			return 0;
3534
3535		free_hash_entry(hash, entry);
3536	} else {
3537		/* Do nothing if it exists */
3538		if (entry)
3539			return 0;
3540
3541		ret = add_hash_entry(hash, rec->ip);
3542	}
3543	return ret;
3544}
3545
3546static int
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3547ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
3548		struct ftrace_glob *mod_g, int exclude_mod)
3549{
3550	char str[KSYM_SYMBOL_LEN];
3551	char *modname;
3552
3553	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
3554
3555	if (mod_g) {
3556		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
3557
3558		/* blank module name to match all modules */
3559		if (!mod_g->len) {
3560			/* blank module globbing: modname xor exclude_mod */
3561			if ((!exclude_mod) != (!modname))
3562				goto func_match;
3563			return 0;
3564		}
3565
3566		/* not matching the module */
3567		if (!modname || !mod_matches) {
3568			if (exclude_mod)
3569				goto func_match;
3570			else
3571				return 0;
3572		}
3573
3574		if (mod_matches && exclude_mod)
3575			return 0;
3576
3577func_match:
3578		/* blank search means to match all funcs in the mod */
3579		if (!func_g->len)
3580			return 1;
3581	}
3582
3583	return ftrace_match(str, func_g);
3584}
3585
3586static int
3587match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3588{
3589	struct ftrace_page *pg;
3590	struct dyn_ftrace *rec;
3591	struct ftrace_glob func_g = { .type = MATCH_FULL };
3592	struct ftrace_glob mod_g = { .type = MATCH_FULL };
3593	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
3594	int exclude_mod = 0;
3595	int found = 0;
3596	int ret;
3597	int clear_filter;
3598
3599	if (func) {
3600		func_g.type = filter_parse_regex(func, len, &func_g.search,
3601						 &clear_filter);
3602		func_g.len = strlen(func_g.search);
3603	}
3604
3605	if (mod) {
3606		mod_g.type = filter_parse_regex(mod, strlen(mod),
3607				&mod_g.search, &exclude_mod);
3608		mod_g.len = strlen(mod_g.search);
3609	}
3610
3611	mutex_lock(&ftrace_lock);
3612
3613	if (unlikely(ftrace_disabled))
3614		goto out_unlock;
3615
 
 
 
 
 
3616	do_for_each_ftrace_rec(pg, rec) {
3617
3618		if (rec->flags & FTRACE_FL_DISABLED)
3619			continue;
3620
3621		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3622			ret = enter_record(hash, rec, clear_filter);
3623			if (ret < 0) {
3624				found = ret;
3625				goto out_unlock;
3626			}
3627			found = 1;
3628		}
3629	} while_for_each_ftrace_rec();
3630 out_unlock:
3631	mutex_unlock(&ftrace_lock);
3632
3633	return found;
3634}
3635
3636static int
3637ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3638{
3639	return match_records(hash, buff, len, NULL);
3640}
3641
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3642
3643/*
3644 * We register the module command as a template to show others how
3645 * to register the a command as well.
3646 */
3647
3648static int
3649ftrace_mod_callback(struct ftrace_hash *hash,
3650		    char *func, char *cmd, char *module, int enable)
3651{
 
3652	int ret;
3653
 
 
 
 
 
3654	/*
3655	 * cmd == 'mod' because we only registered this func
3656	 * for the 'mod' ftrace_func_command.
3657	 * But if you register one func with multiple commands,
3658	 * you can tell which command was used by the cmd
3659	 * parameter.
3660	 */
3661	ret = match_records(hash, func, strlen(func), module);
 
 
3662	if (!ret)
3663		return -EINVAL;
3664	if (ret < 0)
3665		return ret;
3666	return 0;
3667}
3668
3669static struct ftrace_func_command ftrace_mod_cmd = {
3670	.name			= "mod",
3671	.func			= ftrace_mod_callback,
3672};
3673
3674static int __init ftrace_mod_cmd_init(void)
3675{
3676	return register_ftrace_command(&ftrace_mod_cmd);
3677}
3678core_initcall(ftrace_mod_cmd_init);
3679
3680static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
3681				      struct ftrace_ops *op, struct pt_regs *pt_regs)
3682{
3683	struct ftrace_func_probe *entry;
3684	struct hlist_head *hhd;
3685	unsigned long key;
3686
3687	key = hash_long(ip, FTRACE_HASH_BITS);
3688
3689	hhd = &ftrace_func_hash[key];
3690
3691	if (hlist_empty(hhd))
3692		return;
3693
3694	/*
3695	 * Disable preemption for these calls to prevent a RCU grace
3696	 * period. This syncs the hash iteration and freeing of items
3697	 * on the hash. rcu_read_lock is too dangerous here.
3698	 */
3699	preempt_disable_notrace();
3700	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
3701		if (entry->ip == ip)
3702			entry->ops->func(ip, parent_ip, &entry->data);
3703	}
3704	preempt_enable_notrace();
3705}
3706
3707static struct ftrace_ops trace_probe_ops __read_mostly =
3708{
3709	.func		= function_trace_probe_call,
3710	.flags		= FTRACE_OPS_FL_INITIALIZED,
3711	INIT_OPS_HASH(trace_probe_ops)
3712};
3713
3714static int ftrace_probe_registered;
 
 
3715
3716static void __enable_ftrace_function_probe(struct ftrace_ops_hash *old_hash)
 
 
 
 
 
3717{
3718	int ret;
3719	int i;
3720
3721	if (ftrace_probe_registered) {
3722		/* still need to update the function call sites */
3723		if (ftrace_enabled)
3724			ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS,
3725					       old_hash);
3726		return;
3727	}
 
3728
3729	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3730		struct hlist_head *hhd = &ftrace_func_hash[i];
3731		if (hhd->first)
3732			break;
3733	}
3734	/* Nothing registered? */
3735	if (i == FTRACE_FUNC_HASHSIZE)
3736		return;
 
 
 
 
 
 
 
 
3737
3738	ret = ftrace_startup(&trace_probe_ops, 0);
 
 
3739
3740	ftrace_probe_registered = 1;
 
3741}
3742
3743static void __disable_ftrace_function_probe(void)
 
 
 
 
 
 
 
 
 
3744{
3745	int i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3746
3747	if (!ftrace_probe_registered)
3748		return;
3749
3750	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3751		struct hlist_head *hhd = &ftrace_func_hash[i];
3752		if (hhd->first)
3753			return;
 
 
 
 
 
3754	}
 
 
3755
3756	/* no more funcs left */
3757	ftrace_shutdown(&trace_probe_ops, 0);
 
3758
3759	ftrace_probe_registered = 0;
3760}
 
 
 
 
3761
 
 
 
 
 
 
 
 
 
 
 
 
 
3762
3763static void ftrace_free_entry(struct ftrace_func_probe *entry)
3764{
3765	if (entry->ops->free)
3766		entry->ops->free(entry->ops, entry->ip, &entry->data);
3767	kfree(entry);
 
 
3768}
3769
3770int
3771register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3772			      void *data)
 
3773{
3774	struct ftrace_ops_hash old_hash_ops;
3775	struct ftrace_func_probe *entry;
3776	struct ftrace_glob func_g;
3777	struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
3778	struct ftrace_hash *old_hash = *orig_hash;
3779	struct ftrace_hash *hash;
3780	struct ftrace_page *pg;
3781	struct dyn_ftrace *rec;
3782	int not;
3783	unsigned long key;
3784	int count = 0;
 
3785	int ret;
 
3786
3787	func_g.type = filter_parse_regex(glob, strlen(glob),
3788			&func_g.search, &not);
3789	func_g.len = strlen(func_g.search);
3790
3791	/* we do not support '!' for function probes */
3792	if (WARN_ON(not))
3793		return -EINVAL;
3794
3795	mutex_lock(&trace_probe_ops.func_hash->regex_lock);
 
 
3796
3797	old_hash_ops.filter_hash = old_hash;
3798	/* Probes only have filters */
3799	old_hash_ops.notrace_hash = NULL;
3800
3801	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
3802	if (!hash) {
3803		count = -ENOMEM;
3804		goto out;
 
3805	}
3806
3807	if (unlikely(ftrace_disabled)) {
3808		count = -ENODEV;
3809		goto out;
 
 
 
 
 
 
 
3810	}
3811
3812	mutex_lock(&ftrace_lock);
3813
3814	do_for_each_ftrace_rec(pg, rec) {
3815
3816		if (rec->flags & FTRACE_FL_DISABLED)
3817			continue;
 
 
 
3818
3819		if (!ftrace_match_record(rec, &func_g, NULL, 0))
3820			continue;
 
3821
3822		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
3823		if (!entry) {
3824			/* If we did not process any, then return error */
3825			if (!count)
3826				count = -ENOMEM;
3827			goto out_unlock;
3828		}
3829
3830		count++;
3831
3832		entry->data = data;
 
 
3833
3834		/*
3835		 * The caller might want to do something special
3836		 * for each function we find. We call the callback
3837		 * to give the caller an opportunity to do so.
3838		 */
3839		if (ops->init) {
3840			if (ops->init(ops, rec->ip, &entry->data) < 0) {
3841				/* caller does not like this func */
3842				kfree(entry);
3843				continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3844			}
 
3845		}
 
3846
3847		ret = enter_record(hash, rec, 0);
3848		if (ret < 0) {
3849			kfree(entry);
3850			count = ret;
3851			goto out_unlock;
3852		}
3853
3854		entry->ops = ops;
3855		entry->ip = rec->ip;
3856
3857		key = hash_long(entry->ip, FTRACE_HASH_BITS);
3858		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
3859
3860	} while_for_each_ftrace_rec();
 
 
 
 
3861
3862	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
 
 
 
3863
3864	__enable_ftrace_function_probe(&old_hash_ops);
 
3865
3866	if (!ret)
3867		free_ftrace_hash_rcu(old_hash);
3868	else
3869		count = ret;
3870
3871 out_unlock:
3872	mutex_unlock(&ftrace_lock);
 
 
 
3873 out:
3874	mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
3875	free_ftrace_hash(hash);
3876
3877	return count;
3878}
3879
3880enum {
3881	PROBE_TEST_FUNC		= 1,
3882	PROBE_TEST_DATA		= 2
3883};
3884
3885static void
3886__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3887				  void *data, int flags)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3888{
3889	struct ftrace_func_entry *rec_entry;
3890	struct ftrace_func_probe *entry;
3891	struct ftrace_func_probe *p;
3892	struct ftrace_glob func_g;
3893	struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
3894	struct ftrace_hash *old_hash = *orig_hash;
3895	struct list_head free_list;
3896	struct ftrace_hash *hash;
3897	struct hlist_node *tmp;
 
3898	char str[KSYM_SYMBOL_LEN];
3899	int i, ret;
 
 
3900
3901	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
3902		func_g.search = NULL;
3903	else if (glob) {
3904		int not;
3905
3906		func_g.type = filter_parse_regex(glob, strlen(glob),
3907						 &func_g.search, &not);
3908		func_g.len = strlen(func_g.search);
3909		func_g.search = glob;
3910
3911		/* we do not support '!' for function probes */
3912		if (WARN_ON(not))
3913			return;
3914	}
3915
3916	mutex_lock(&trace_probe_ops.func_hash->regex_lock);
 
 
 
 
 
 
 
3917
3918	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3919	if (!hash)
3920		/* Hmm, should report this somehow */
3921		goto out_unlock;
3922
3923	INIT_LIST_HEAD(&free_list);
3924
3925	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3926		struct hlist_head *hhd = &ftrace_func_hash[i];
3927
3928		hlist_for_each_entry_safe(entry, tmp, hhd, node) {
3929
3930			/* break up if statements for readability */
3931			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
3932				continue;
3933
3934			if ((flags & PROBE_TEST_DATA) && entry->data != data)
3935				continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3936
3937			/* do this last, since it is the most expensive */
3938			if (func_g.search) {
3939				kallsyms_lookup(entry->ip, NULL, NULL,
3940						NULL, str);
3941				if (!ftrace_match(str, &func_g))
3942					continue;
3943			}
3944
3945			rec_entry = ftrace_lookup_ip(hash, entry->ip);
3946			/* It is possible more than one entry had this ip */
3947			if (rec_entry)
3948				free_hash_entry(hash, rec_entry);
3949
3950			hlist_del_rcu(&entry->node);
3951			list_add(&entry->free_list, &free_list);
3952		}
3953	}
 
 
 
 
 
 
 
3954	mutex_lock(&ftrace_lock);
3955	__disable_ftrace_function_probe();
3956	/*
3957	 * Remove after the disable is called. Otherwise, if the last
3958	 * probe is removed, a null hash means *all enabled*.
3959	 */
3960	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
3961	synchronize_sched();
3962	if (!ret)
3963		free_ftrace_hash_rcu(old_hash);
3964
3965	list_for_each_entry_safe(entry, p, &free_list, free_list) {
3966		list_del(&entry->free_list);
3967		ftrace_free_entry(entry);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3968	}
3969	mutex_unlock(&ftrace_lock);
3970
3971 out_unlock:
3972	mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
3973	free_ftrace_hash(hash);
3974}
3975
3976void
3977unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3978				void *data)
3979{
3980	__unregister_ftrace_function_probe(glob, ops, data,
3981					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
3982}
3983
3984void
3985unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
3986{
3987	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
 
3988}
3989
3990void unregister_ftrace_function_probe_all(char *glob)
3991{
3992	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
 
 
 
3993}
3994
3995static LIST_HEAD(ftrace_commands);
3996static DEFINE_MUTEX(ftrace_cmd_mutex);
3997
3998/*
3999 * Currently we only register ftrace commands from __init, so mark this
4000 * __init too.
4001 */
4002__init int register_ftrace_command(struct ftrace_func_command *cmd)
4003{
4004	struct ftrace_func_command *p;
4005	int ret = 0;
4006
4007	mutex_lock(&ftrace_cmd_mutex);
4008	list_for_each_entry(p, &ftrace_commands, list) {
4009		if (strcmp(cmd->name, p->name) == 0) {
4010			ret = -EBUSY;
4011			goto out_unlock;
4012		}
4013	}
4014	list_add(&cmd->list, &ftrace_commands);
4015 out_unlock:
4016	mutex_unlock(&ftrace_cmd_mutex);
4017
4018	return ret;
4019}
4020
4021/*
4022 * Currently we only unregister ftrace commands from __init, so mark
4023 * this __init too.
4024 */
4025__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4026{
4027	struct ftrace_func_command *p, *n;
4028	int ret = -ENODEV;
4029
4030	mutex_lock(&ftrace_cmd_mutex);
4031	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
4032		if (strcmp(cmd->name, p->name) == 0) {
4033			ret = 0;
4034			list_del_init(&p->list);
4035			goto out_unlock;
4036		}
4037	}
4038 out_unlock:
4039	mutex_unlock(&ftrace_cmd_mutex);
4040
4041	return ret;
4042}
4043
4044static int ftrace_process_regex(struct ftrace_hash *hash,
4045				char *buff, int len, int enable)
4046{
 
 
4047	char *func, *command, *next = buff;
4048	struct ftrace_func_command *p;
4049	int ret = -EINVAL;
4050
4051	func = strsep(&next, ":");
4052
4053	if (!next) {
4054		ret = ftrace_match_records(hash, func, len);
4055		if (!ret)
4056			ret = -EINVAL;
4057		if (ret < 0)
4058			return ret;
4059		return 0;
4060	}
4061
4062	/* command found */
4063
4064	command = strsep(&next, ":");
4065
4066	mutex_lock(&ftrace_cmd_mutex);
4067	list_for_each_entry(p, &ftrace_commands, list) {
4068		if (strcmp(p->name, command) == 0) {
4069			ret = p->func(hash, func, command, next, enable);
4070			goto out_unlock;
4071		}
4072	}
4073 out_unlock:
4074	mutex_unlock(&ftrace_cmd_mutex);
4075
4076	return ret;
4077}
4078
4079static ssize_t
4080ftrace_regex_write(struct file *file, const char __user *ubuf,
4081		   size_t cnt, loff_t *ppos, int enable)
4082{
4083	struct ftrace_iterator *iter;
4084	struct trace_parser *parser;
4085	ssize_t ret, read;
4086
4087	if (!cnt)
4088		return 0;
4089
4090	if (file->f_mode & FMODE_READ) {
4091		struct seq_file *m = file->private_data;
4092		iter = m->private;
4093	} else
4094		iter = file->private_data;
4095
4096	if (unlikely(ftrace_disabled))
4097		return -ENODEV;
4098
4099	/* iter->hash is a local copy, so we don't need regex_lock */
4100
4101	parser = &iter->parser;
4102	read = trace_get_user(parser, ubuf, cnt, ppos);
4103
4104	if (read >= 0 && trace_parser_loaded(parser) &&
4105	    !trace_parser_cont(parser)) {
4106		ret = ftrace_process_regex(iter->hash, parser->buffer,
4107					   parser->idx, enable);
4108		trace_parser_clear(parser);
4109		if (ret < 0)
4110			goto out;
4111	}
4112
4113	ret = read;
4114 out:
4115	return ret;
4116}
4117
4118ssize_t
4119ftrace_filter_write(struct file *file, const char __user *ubuf,
4120		    size_t cnt, loff_t *ppos)
4121{
4122	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
4123}
4124
4125ssize_t
4126ftrace_notrace_write(struct file *file, const char __user *ubuf,
4127		     size_t cnt, loff_t *ppos)
4128{
4129	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
4130}
4131
4132static int
4133ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
4134{
4135	struct ftrace_func_entry *entry;
4136
4137	if (!ftrace_location(ip))
4138		return -EINVAL;
4139
4140	if (remove) {
4141		entry = ftrace_lookup_ip(hash, ip);
4142		if (!entry)
4143			return -ENOENT;
4144		free_hash_entry(hash, entry);
4145		return 0;
4146	}
4147
4148	return add_hash_entry(hash, ip);
4149}
4150
4151static void ftrace_ops_update_code(struct ftrace_ops *ops,
4152				   struct ftrace_ops_hash *old_hash)
4153{
4154	struct ftrace_ops *op;
4155
4156	if (!ftrace_enabled)
4157		return;
4158
4159	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
4160		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
4161		return;
4162	}
4163
4164	/*
4165	 * If this is the shared global_ops filter, then we need to
4166	 * check if there is another ops that shares it, is enabled.
4167	 * If so, we still need to run the modify code.
4168	 */
4169	if (ops->func_hash != &global_ops.local_hash)
4170		return;
4171
4172	do_for_each_ftrace_op(op, ftrace_ops_list) {
4173		if (op->func_hash == &global_ops.local_hash &&
4174		    op->flags & FTRACE_OPS_FL_ENABLED) {
4175			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
4176			/* Only need to do this once */
4177			return;
4178		}
4179	} while_for_each_ftrace_op(op);
4180}
4181
4182static int
4183ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
4184		unsigned long ip, int remove, int reset, int enable)
4185{
4186	struct ftrace_hash **orig_hash;
4187	struct ftrace_ops_hash old_hash_ops;
4188	struct ftrace_hash *old_hash;
4189	struct ftrace_hash *hash;
4190	int ret;
4191
4192	if (unlikely(ftrace_disabled))
4193		return -ENODEV;
4194
4195	mutex_lock(&ops->func_hash->regex_lock);
4196
4197	if (enable)
4198		orig_hash = &ops->func_hash->filter_hash;
4199	else
4200		orig_hash = &ops->func_hash->notrace_hash;
4201
4202	if (reset)
4203		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4204	else
4205		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
4206
4207	if (!hash) {
4208		ret = -ENOMEM;
4209		goto out_regex_unlock;
4210	}
4211
4212	if (buf && !ftrace_match_records(hash, buf, len)) {
4213		ret = -EINVAL;
4214		goto out_regex_unlock;
4215	}
4216	if (ip) {
4217		ret = ftrace_match_addr(hash, ip, remove);
4218		if (ret < 0)
4219			goto out_regex_unlock;
4220	}
4221
4222	mutex_lock(&ftrace_lock);
4223	old_hash = *orig_hash;
4224	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
4225	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
4226	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
4227	if (!ret) {
4228		ftrace_ops_update_code(ops, &old_hash_ops);
4229		free_ftrace_hash_rcu(old_hash);
4230	}
4231	mutex_unlock(&ftrace_lock);
4232
4233 out_regex_unlock:
4234	mutex_unlock(&ops->func_hash->regex_lock);
4235
4236	free_ftrace_hash(hash);
4237	return ret;
4238}
4239
4240static int
4241ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
4242		int reset, int enable)
4243{
4244	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4245}
 
 
4246
4247/**
4248 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
4249 * @ops - the ops to set the filter with
4250 * @ip - the address to add to or remove from the filter.
4251 * @remove - non zero to remove the ip from the filter
4252 * @reset - non zero to reset all filters before applying this filter.
4253 *
4254 * Filters denote which functions should be enabled when tracing is enabled
4255 * If @ip is NULL, it failes to update filter.
4256 */
4257int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
4258			 int remove, int reset)
4259{
4260	ftrace_ops_init(ops);
4261	return ftrace_set_addr(ops, ip, remove, reset, 1);
4262}
4263EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
4264
4265/**
4266 * ftrace_ops_set_global_filter - setup ops to use global filters
4267 * @ops - the ops which will use the global filters
4268 *
4269 * ftrace users who need global function trace filtering should call this.
4270 * It can set the global filter only if ops were not initialized before.
4271 */
4272void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
4273{
4274	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
4275		return;
4276
4277	ftrace_ops_init(ops);
4278	ops->func_hash = &global_ops.local_hash;
4279}
4280EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
4281
4282static int
4283ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
4284		 int reset, int enable)
4285{
4286	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
4287}
4288
4289/**
4290 * ftrace_set_filter - set a function to filter on in ftrace
4291 * @ops - the ops to set the filter with
4292 * @buf - the string that holds the function filter text.
4293 * @len - the length of the string.
4294 * @reset - non zero to reset all filters before applying this filter.
4295 *
4296 * Filters denote which functions should be enabled when tracing is enabled.
4297 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
4298 */
4299int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4300		       int len, int reset)
4301{
4302	ftrace_ops_init(ops);
4303	return ftrace_set_regex(ops, buf, len, reset, 1);
4304}
4305EXPORT_SYMBOL_GPL(ftrace_set_filter);
4306
4307/**
4308 * ftrace_set_notrace - set a function to not trace in ftrace
4309 * @ops - the ops to set the notrace filter with
4310 * @buf - the string that holds the function notrace text.
4311 * @len - the length of the string.
4312 * @reset - non zero to reset all filters before applying this filter.
4313 *
4314 * Notrace Filters denote which functions should not be enabled when tracing
4315 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
4316 * for tracing.
4317 */
4318int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4319			int len, int reset)
4320{
4321	ftrace_ops_init(ops);
4322	return ftrace_set_regex(ops, buf, len, reset, 0);
4323}
4324EXPORT_SYMBOL_GPL(ftrace_set_notrace);
4325/**
4326 * ftrace_set_global_filter - set a function to filter on with global tracers
4327 * @buf - the string that holds the function filter text.
4328 * @len - the length of the string.
4329 * @reset - non zero to reset all filters before applying this filter.
4330 *
4331 * Filters denote which functions should be enabled when tracing is enabled.
4332 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
4333 */
4334void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4335{
4336	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4337}
4338EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4339
4340/**
4341 * ftrace_set_global_notrace - set a function to not trace with global tracers
4342 * @buf - the string that holds the function notrace text.
4343 * @len - the length of the string.
4344 * @reset - non zero to reset all filters before applying this filter.
4345 *
4346 * Notrace Filters denote which functions should not be enabled when tracing
4347 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
4348 * for tracing.
4349 */
4350void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4351{
4352	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4353}
4354EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4355
4356/*
4357 * command line interface to allow users to set filters on boot up.
4358 */
4359#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
4360static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4361static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
4362
4363/* Used by function selftest to not test if filter is set */
4364bool ftrace_filter_param __initdata;
4365
4366static int __init set_ftrace_notrace(char *str)
4367{
4368	ftrace_filter_param = true;
4369	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4370	return 1;
4371}
4372__setup("ftrace_notrace=", set_ftrace_notrace);
4373
4374static int __init set_ftrace_filter(char *str)
4375{
4376	ftrace_filter_param = true;
4377	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4378	return 1;
4379}
4380__setup("ftrace_filter=", set_ftrace_filter);
4381
4382#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4383static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4384static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4385static int ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer);
4386
4387static unsigned long save_global_trampoline;
4388static unsigned long save_global_flags;
4389
4390static int __init set_graph_function(char *str)
4391{
4392	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4393	return 1;
4394}
4395__setup("ftrace_graph_filter=", set_graph_function);
4396
4397static int __init set_graph_notrace_function(char *str)
4398{
4399	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
4400	return 1;
4401}
4402__setup("ftrace_graph_notrace=", set_graph_notrace_function);
4403
 
 
 
 
 
 
 
 
 
4404static void __init set_ftrace_early_graph(char *buf, int enable)
4405{
4406	int ret;
4407	char *func;
4408	unsigned long *table = ftrace_graph_funcs;
4409	int *count = &ftrace_graph_count;
4410
4411	if (!enable) {
4412		table = ftrace_graph_notrace_funcs;
4413		count = &ftrace_graph_notrace_count;
4414	}
4415
4416	while (buf) {
4417		func = strsep(&buf, ",");
4418		/* we allow only one expression at a time */
4419		ret = ftrace_set_func(table, count, FTRACE_GRAPH_MAX_FUNCS, func);
4420		if (ret)
4421			printk(KERN_DEBUG "ftrace: function %s not "
4422					  "traceable\n", func);
4423	}
 
 
 
 
 
4424}
4425#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4426
4427void __init
4428ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
4429{
4430	char *func;
4431
4432	ftrace_ops_init(ops);
4433
4434	while (buf) {
4435		func = strsep(&buf, ",");
4436		ftrace_set_regex(ops, func, strlen(func), 0, enable);
4437	}
4438}
4439
4440static void __init set_ftrace_early_filters(void)
4441{
4442	if (ftrace_filter_buf[0])
4443		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
4444	if (ftrace_notrace_buf[0])
4445		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
4446#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4447	if (ftrace_graph_buf[0])
4448		set_ftrace_early_graph(ftrace_graph_buf, 1);
4449	if (ftrace_graph_notrace_buf[0])
4450		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
4451#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4452}
4453
4454int ftrace_regex_release(struct inode *inode, struct file *file)
4455{
4456	struct seq_file *m = (struct seq_file *)file->private_data;
4457	struct ftrace_ops_hash old_hash_ops;
4458	struct ftrace_iterator *iter;
4459	struct ftrace_hash **orig_hash;
4460	struct ftrace_hash *old_hash;
4461	struct trace_parser *parser;
4462	int filter_hash;
4463	int ret;
4464
4465	if (file->f_mode & FMODE_READ) {
4466		iter = m->private;
4467		seq_release(inode, file);
4468	} else
4469		iter = file->private_data;
4470
4471	parser = &iter->parser;
4472	if (trace_parser_loaded(parser)) {
4473		parser->buffer[parser->idx] = 0;
4474		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
 
 
4475	}
4476
4477	trace_parser_put(parser);
4478
4479	mutex_lock(&iter->ops->func_hash->regex_lock);
4480
4481	if (file->f_mode & FMODE_WRITE) {
4482		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
4483
4484		if (filter_hash)
4485			orig_hash = &iter->ops->func_hash->filter_hash;
4486		else
 
 
4487			orig_hash = &iter->ops->func_hash->notrace_hash;
4488
4489		mutex_lock(&ftrace_lock);
4490		old_hash = *orig_hash;
4491		old_hash_ops.filter_hash = iter->ops->func_hash->filter_hash;
4492		old_hash_ops.notrace_hash = iter->ops->func_hash->notrace_hash;
4493		ret = ftrace_hash_move(iter->ops, filter_hash,
4494				       orig_hash, iter->hash);
4495		if (!ret) {
4496			ftrace_ops_update_code(iter->ops, &old_hash_ops);
4497			free_ftrace_hash_rcu(old_hash);
4498		}
4499		mutex_unlock(&ftrace_lock);
 
 
 
4500	}
4501
4502	mutex_unlock(&iter->ops->func_hash->regex_lock);
4503	free_ftrace_hash(iter->hash);
 
 
4504	kfree(iter);
4505
4506	return 0;
4507}
4508
4509static const struct file_operations ftrace_avail_fops = {
4510	.open = ftrace_avail_open,
4511	.read = seq_read,
4512	.llseek = seq_lseek,
4513	.release = seq_release_private,
4514};
4515
4516static const struct file_operations ftrace_enabled_fops = {
4517	.open = ftrace_enabled_open,
4518	.read = seq_read,
4519	.llseek = seq_lseek,
4520	.release = seq_release_private,
4521};
4522
4523static const struct file_operations ftrace_filter_fops = {
4524	.open = ftrace_filter_open,
4525	.read = seq_read,
4526	.write = ftrace_filter_write,
4527	.llseek = tracing_lseek,
4528	.release = ftrace_regex_release,
4529};
4530
4531static const struct file_operations ftrace_notrace_fops = {
4532	.open = ftrace_notrace_open,
4533	.read = seq_read,
4534	.write = ftrace_notrace_write,
4535	.llseek = tracing_lseek,
4536	.release = ftrace_regex_release,
4537};
4538
4539#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4540
4541static DEFINE_MUTEX(graph_lock);
4542
4543int ftrace_graph_count;
4544int ftrace_graph_notrace_count;
4545unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
4546unsigned long ftrace_graph_notrace_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
 
 
 
 
 
4547
4548struct ftrace_graph_data {
4549	unsigned long *table;
4550	size_t size;
4551	int *count;
4552	const struct seq_operations *seq_ops;
 
 
 
4553};
4554
4555static void *
4556__g_next(struct seq_file *m, loff_t *pos)
4557{
4558	struct ftrace_graph_data *fgd = m->private;
 
 
 
4559
4560	if (*pos >= *fgd->count)
4561		return NULL;
4562	return &fgd->table[*pos];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4563}
4564
4565static void *
4566g_next(struct seq_file *m, void *v, loff_t *pos)
4567{
4568	(*pos)++;
4569	return __g_next(m, pos);
4570}
4571
4572static void *g_start(struct seq_file *m, loff_t *pos)
4573{
4574	struct ftrace_graph_data *fgd = m->private;
4575
4576	mutex_lock(&graph_lock);
4577
 
 
 
 
 
 
 
4578	/* Nothing, tell g_show to print all functions are enabled */
4579	if (!*fgd->count && !*pos)
4580		return (void *)1;
4581
 
 
4582	return __g_next(m, pos);
4583}
4584
4585static void g_stop(struct seq_file *m, void *p)
4586{
4587	mutex_unlock(&graph_lock);
4588}
4589
4590static int g_show(struct seq_file *m, void *v)
4591{
4592	unsigned long *ptr = v;
4593
4594	if (!ptr)
4595		return 0;
4596
4597	if (ptr == (unsigned long *)1) {
4598		struct ftrace_graph_data *fgd = m->private;
4599
4600		if (fgd->table == ftrace_graph_funcs)
4601			seq_puts(m, "#### all functions enabled ####\n");
4602		else
4603			seq_puts(m, "#### no functions disabled ####\n");
4604		return 0;
4605	}
4606
4607	seq_printf(m, "%ps\n", (void *)*ptr);
4608
4609	return 0;
4610}
4611
4612static const struct seq_operations ftrace_graph_seq_ops = {
4613	.start = g_start,
4614	.next = g_next,
4615	.stop = g_stop,
4616	.show = g_show,
4617};
4618
4619static int
4620__ftrace_graph_open(struct inode *inode, struct file *file,
4621		    struct ftrace_graph_data *fgd)
4622{
4623	int ret = 0;
 
4624
4625	mutex_lock(&graph_lock);
4626	if ((file->f_mode & FMODE_WRITE) &&
4627	    (file->f_flags & O_TRUNC)) {
4628		*fgd->count = 0;
4629		memset(fgd->table, 0, fgd->size * sizeof(*fgd->table));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4630	}
4631	mutex_unlock(&graph_lock);
4632
4633	if (file->f_mode & FMODE_READ) {
4634		ret = seq_open(file, fgd->seq_ops);
4635		if (!ret) {
4636			struct seq_file *m = file->private_data;
4637			m->private = fgd;
 
 
 
 
4638		}
4639	} else
4640		file->private_data = fgd;
4641
 
 
 
 
 
 
 
 
 
 
 
 
 
4642	return ret;
4643}
4644
4645static int
4646ftrace_graph_open(struct inode *inode, struct file *file)
4647{
4648	struct ftrace_graph_data *fgd;
 
4649
4650	if (unlikely(ftrace_disabled))
4651		return -ENODEV;
4652
4653	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
4654	if (fgd == NULL)
4655		return -ENOMEM;
4656
4657	fgd->table = ftrace_graph_funcs;
4658	fgd->size = FTRACE_GRAPH_MAX_FUNCS;
4659	fgd->count = &ftrace_graph_count;
 
 
4660	fgd->seq_ops = &ftrace_graph_seq_ops;
4661
4662	return __ftrace_graph_open(inode, file, fgd);
 
 
 
 
 
4663}
4664
4665static int
4666ftrace_graph_notrace_open(struct inode *inode, struct file *file)
4667{
4668	struct ftrace_graph_data *fgd;
 
4669
4670	if (unlikely(ftrace_disabled))
4671		return -ENODEV;
4672
4673	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
4674	if (fgd == NULL)
4675		return -ENOMEM;
4676
4677	fgd->table = ftrace_graph_notrace_funcs;
4678	fgd->size = FTRACE_GRAPH_MAX_FUNCS;
4679	fgd->count = &ftrace_graph_notrace_count;
 
 
4680	fgd->seq_ops = &ftrace_graph_seq_ops;
4681
4682	return __ftrace_graph_open(inode, file, fgd);
 
 
 
 
 
4683}
4684
4685static int
4686ftrace_graph_release(struct inode *inode, struct file *file)
4687{
 
 
 
 
 
4688	if (file->f_mode & FMODE_READ) {
4689		struct seq_file *m = file->private_data;
4690
4691		kfree(m->private);
4692		seq_release(inode, file);
4693	} else {
4694		kfree(file->private_data);
4695	}
4696
4697	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4698}
4699
4700static int
4701ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer)
4702{
4703	struct ftrace_glob func_g;
4704	struct dyn_ftrace *rec;
4705	struct ftrace_page *pg;
 
4706	int fail = 1;
4707	int not;
4708	bool exists;
4709	int i;
4710
4711	/* decode regex */
4712	func_g.type = filter_parse_regex(buffer, strlen(buffer),
4713					 &func_g.search, &not);
4714	if (!not && *idx >= size)
4715		return -EBUSY;
4716
4717	func_g.len = strlen(func_g.search);
4718
4719	mutex_lock(&ftrace_lock);
4720
4721	if (unlikely(ftrace_disabled)) {
4722		mutex_unlock(&ftrace_lock);
4723		return -ENODEV;
4724	}
4725
4726	do_for_each_ftrace_rec(pg, rec) {
4727
4728		if (rec->flags & FTRACE_FL_DISABLED)
4729			continue;
4730
4731		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
4732			/* if it is in the array */
4733			exists = false;
4734			for (i = 0; i < *idx; i++) {
4735				if (array[i] == rec->ip) {
4736					exists = true;
4737					break;
4738				}
4739			}
4740
4741			if (!not) {
4742				fail = 0;
4743				if (!exists) {
4744					array[(*idx)++] = rec->ip;
4745					if (*idx >= size)
4746						goto out;
4747				}
4748			} else {
4749				if (exists) {
4750					array[i] = array[--(*idx)];
4751					array[*idx] = 0;
4752					fail = 0;
4753				}
4754			}
4755		}
4756	} while_for_each_ftrace_rec();
4757out:
4758	mutex_unlock(&ftrace_lock);
4759
4760	if (fail)
4761		return -EINVAL;
4762
4763	return 0;
4764}
4765
4766static ssize_t
4767ftrace_graph_write(struct file *file, const char __user *ubuf,
4768		   size_t cnt, loff_t *ppos)
4769{
4770	struct trace_parser parser;
4771	ssize_t read, ret = 0;
4772	struct ftrace_graph_data *fgd = file->private_data;
 
4773
4774	if (!cnt)
4775		return 0;
4776
4777	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX))
4778		return -ENOMEM;
4779
4780	read = trace_get_user(&parser, ubuf, cnt, ppos);
 
4781
4782	if (read >= 0 && trace_parser_loaded((&parser))) {
4783		parser.buffer[parser.idx] = 0;
4784
4785		mutex_lock(&graph_lock);
4786
4787		/* we allow only one expression at a time */
4788		ret = ftrace_set_func(fgd->table, fgd->count, fgd->size,
4789				      parser.buffer);
4790
4791		mutex_unlock(&graph_lock);
 
 
4792	}
4793
4794	if (!ret)
4795		ret = read;
4796
4797	trace_parser_put(&parser);
4798
4799	return ret;
4800}
4801
4802static const struct file_operations ftrace_graph_fops = {
4803	.open		= ftrace_graph_open,
4804	.read		= seq_read,
4805	.write		= ftrace_graph_write,
4806	.llseek		= tracing_lseek,
4807	.release	= ftrace_graph_release,
4808};
4809
4810static const struct file_operations ftrace_graph_notrace_fops = {
4811	.open		= ftrace_graph_notrace_open,
4812	.read		= seq_read,
4813	.write		= ftrace_graph_write,
4814	.llseek		= tracing_lseek,
4815	.release	= ftrace_graph_release,
4816};
4817#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4818
4819void ftrace_create_filter_files(struct ftrace_ops *ops,
4820				struct dentry *parent)
4821{
4822
4823	trace_create_file("set_ftrace_filter", 0644, parent,
4824			  ops, &ftrace_filter_fops);
4825
4826	trace_create_file("set_ftrace_notrace", 0644, parent,
4827			  ops, &ftrace_notrace_fops);
4828}
4829
4830/*
4831 * The name "destroy_filter_files" is really a misnomer. Although
4832 * in the future, it may actualy delete the files, but this is
4833 * really intended to make sure the ops passed in are disabled
4834 * and that when this function returns, the caller is free to
4835 * free the ops.
4836 *
4837 * The "destroy" name is only to match the "create" name that this
4838 * should be paired with.
4839 */
4840void ftrace_destroy_filter_files(struct ftrace_ops *ops)
4841{
4842	mutex_lock(&ftrace_lock);
4843	if (ops->flags & FTRACE_OPS_FL_ENABLED)
4844		ftrace_shutdown(ops, 0);
4845	ops->flags |= FTRACE_OPS_FL_DELETED;
 
4846	mutex_unlock(&ftrace_lock);
4847}
4848
4849static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
4850{
4851
4852	trace_create_file("available_filter_functions", 0444,
4853			d_tracer, NULL, &ftrace_avail_fops);
4854
4855	trace_create_file("enabled_functions", 0444,
4856			d_tracer, NULL, &ftrace_enabled_fops);
4857
4858	ftrace_create_filter_files(&global_ops, d_tracer);
4859
4860#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4861	trace_create_file("set_graph_function", 0444, d_tracer,
4862				    NULL,
4863				    &ftrace_graph_fops);
4864	trace_create_file("set_graph_notrace", 0444, d_tracer,
4865				    NULL,
4866				    &ftrace_graph_notrace_fops);
4867#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4868
4869	return 0;
4870}
4871
4872static int ftrace_cmp_ips(const void *a, const void *b)
4873{
4874	const unsigned long *ipa = a;
4875	const unsigned long *ipb = b;
4876
4877	if (*ipa > *ipb)
4878		return 1;
4879	if (*ipa < *ipb)
4880		return -1;
4881	return 0;
4882}
4883
4884static int ftrace_process_locs(struct module *mod,
4885			       unsigned long *start,
4886			       unsigned long *end)
4887{
4888	struct ftrace_page *start_pg;
4889	struct ftrace_page *pg;
4890	struct dyn_ftrace *rec;
4891	unsigned long count;
4892	unsigned long *p;
4893	unsigned long addr;
4894	unsigned long flags = 0; /* Shut up gcc */
4895	int ret = -ENOMEM;
4896
4897	count = end - start;
4898
4899	if (!count)
4900		return 0;
4901
4902	sort(start, count, sizeof(*start),
4903	     ftrace_cmp_ips, NULL);
4904
4905	start_pg = ftrace_allocate_pages(count);
4906	if (!start_pg)
4907		return -ENOMEM;
4908
4909	mutex_lock(&ftrace_lock);
4910
4911	/*
4912	 * Core and each module needs their own pages, as
4913	 * modules will free them when they are removed.
4914	 * Force a new page to be allocated for modules.
4915	 */
4916	if (!mod) {
4917		WARN_ON(ftrace_pages || ftrace_pages_start);
4918		/* First initialization */
4919		ftrace_pages = ftrace_pages_start = start_pg;
4920	} else {
4921		if (!ftrace_pages)
4922			goto out;
4923
4924		if (WARN_ON(ftrace_pages->next)) {
4925			/* Hmm, we have free pages? */
4926			while (ftrace_pages->next)
4927				ftrace_pages = ftrace_pages->next;
4928		}
4929
4930		ftrace_pages->next = start_pg;
4931	}
4932
4933	p = start;
4934	pg = start_pg;
4935	while (p < end) {
 
4936		addr = ftrace_call_adjust(*p++);
4937		/*
4938		 * Some architecture linkers will pad between
4939		 * the different mcount_loc sections of different
4940		 * object files to satisfy alignments.
4941		 * Skip any NULL pointers.
4942		 */
4943		if (!addr)
4944			continue;
4945
4946		if (pg->index == pg->size) {
 
4947			/* We should have allocated enough */
4948			if (WARN_ON(!pg->next))
4949				break;
4950			pg = pg->next;
4951		}
4952
4953		rec = &pg->records[pg->index++];
4954		rec->ip = addr;
4955	}
4956
4957	/* We should have used all pages */
4958	WARN_ON(pg->next);
4959
4960	/* Assign the last page to ftrace_pages */
4961	ftrace_pages = pg;
4962
4963	/*
4964	 * We only need to disable interrupts on start up
4965	 * because we are modifying code that an interrupt
4966	 * may execute, and the modification is not atomic.
4967	 * But for modules, nothing runs the code we modify
4968	 * until we are finished with it, and there's no
4969	 * reason to cause large interrupt latencies while we do it.
4970	 */
4971	if (!mod)
4972		local_irq_save(flags);
4973	ftrace_update_code(mod, start_pg);
4974	if (!mod)
4975		local_irq_restore(flags);
4976	ret = 0;
4977 out:
4978	mutex_unlock(&ftrace_lock);
4979
4980	return ret;
4981}
4982
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4983#ifdef CONFIG_MODULES
4984
4985#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
4986
 
 
4987static int referenced_filters(struct dyn_ftrace *rec)
4988{
4989	struct ftrace_ops *ops;
4990	int cnt = 0;
4991
4992	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
4993		if (ops_references_rec(ops, rec))
4994		    cnt++;
 
 
 
 
 
 
 
 
 
 
 
4995	}
4996
4997	return cnt;
4998}
4999
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5000void ftrace_release_mod(struct module *mod)
5001{
 
 
5002	struct dyn_ftrace *rec;
5003	struct ftrace_page **last_pg;
 
5004	struct ftrace_page *pg;
5005	int order;
5006
5007	mutex_lock(&ftrace_lock);
5008
5009	if (ftrace_disabled)
5010		goto out_unlock;
5011
 
 
 
 
 
 
 
 
5012	/*
5013	 * Each module has its own ftrace_pages, remove
5014	 * them from the list.
5015	 */
5016	last_pg = &ftrace_pages_start;
5017	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
5018		rec = &pg->records[0];
5019		if (within_module_core(rec->ip, mod)) {
 
5020			/*
5021			 * As core pages are first, the first
5022			 * page should never be a module page.
5023			 */
5024			if (WARN_ON(pg == ftrace_pages_start))
5025				goto out_unlock;
5026
5027			/* Check if we are deleting the last page */
5028			if (pg == ftrace_pages)
5029				ftrace_pages = next_to_ftrace_page(last_pg);
5030
 
5031			*last_pg = pg->next;
5032			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
5033			free_pages((unsigned long)pg->records, order);
5034			kfree(pg);
5035		} else
5036			last_pg = &pg->next;
5037	}
5038 out_unlock:
5039	mutex_unlock(&ftrace_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5040}
5041
5042void ftrace_module_enable(struct module *mod)
5043{
5044	struct dyn_ftrace *rec;
5045	struct ftrace_page *pg;
5046
5047	mutex_lock(&ftrace_lock);
5048
5049	if (ftrace_disabled)
5050		goto out_unlock;
5051
5052	/*
5053	 * If the tracing is enabled, go ahead and enable the record.
5054	 *
5055	 * The reason not to enable the record immediatelly is the
5056	 * inherent check of ftrace_make_nop/ftrace_make_call for
5057	 * correct previous instructions.  Making first the NOP
5058	 * conversion puts the module to the correct state, thus
5059	 * passing the ftrace_make_call check.
5060	 *
5061	 * We also delay this to after the module code already set the
5062	 * text to read-only, as we now need to set it back to read-write
5063	 * so that we can modify the text.
5064	 */
5065	if (ftrace_start_up)
5066		ftrace_arch_code_modify_prepare();
5067
5068	do_for_each_ftrace_rec(pg, rec) {
5069		int cnt;
5070		/*
5071		 * do_for_each_ftrace_rec() is a double loop.
5072		 * module text shares the pg. If a record is
5073		 * not part of this module, then skip this pg,
5074		 * which the "break" will do.
5075		 */
5076		if (!within_module_core(rec->ip, mod))
 
5077			break;
5078
5079		cnt = 0;
5080
5081		/*
5082		 * When adding a module, we need to check if tracers are
5083		 * currently enabled and if they are, and can trace this record,
5084		 * we need to enable the module functions as well as update the
5085		 * reference counts for those function records.
5086		 */
5087		if (ftrace_start_up)
5088			cnt += referenced_filters(rec);
5089
5090		/* This clears FTRACE_FL_DISABLED */
5091		rec->flags = cnt;
5092
5093		if (ftrace_start_up && cnt) {
5094			int failed = __ftrace_replace_code(rec, 1);
5095			if (failed) {
5096				ftrace_bug(failed, rec);
5097				goto out_loop;
5098			}
5099		}
5100
5101	} while_for_each_ftrace_rec();
5102
5103 out_loop:
5104	if (ftrace_start_up)
5105		ftrace_arch_code_modify_post_process();
5106
5107 out_unlock:
5108	mutex_unlock(&ftrace_lock);
 
 
5109}
5110
5111void ftrace_module_init(struct module *mod)
5112{
5113	if (ftrace_disabled || !mod->num_ftrace_callsites)
5114		return;
5115
5116	ftrace_process_locs(mod, mod->ftrace_callsites,
5117			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5118}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5119#endif /* CONFIG_MODULES */
5120
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5121void __init ftrace_init(void)
5122{
5123	extern unsigned long __start_mcount_loc[];
5124	extern unsigned long __stop_mcount_loc[];
5125	unsigned long count, flags;
5126	int ret;
5127
5128	local_irq_save(flags);
5129	ret = ftrace_dyn_arch_init();
5130	local_irq_restore(flags);
5131	if (ret)
5132		goto failed;
5133
5134	count = __stop_mcount_loc - __start_mcount_loc;
5135	if (!count) {
5136		pr_info("ftrace: No functions to be traced?\n");
5137		goto failed;
5138	}
5139
5140	pr_info("ftrace: allocating %ld entries in %ld pages\n",
5141		count, count / ENTRIES_PER_PAGE + 1);
5142
5143	last_ftrace_enabled = ftrace_enabled = 1;
5144
5145	ret = ftrace_process_locs(NULL,
5146				  __start_mcount_loc,
5147				  __stop_mcount_loc);
5148
 
 
 
5149	set_ftrace_early_filters();
5150
5151	return;
5152 failed:
5153	ftrace_disabled = 1;
5154}
5155
5156/* Do nothing if arch does not support this */
5157void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
5158{
5159}
5160
5161static void ftrace_update_trampoline(struct ftrace_ops *ops)
5162{
5163
5164/*
5165 * Currently there's no safe way to free a trampoline when the kernel
5166 * is configured with PREEMPT. That is because a task could be preempted
5167 * when it jumped to the trampoline, it may be preempted for a long time
5168 * depending on the system load, and currently there's no way to know
5169 * when it will be off the trampoline. If the trampoline is freed
5170 * too early, when the task runs again, it will be executing on freed
5171 * memory and crash.
5172 */
5173#ifdef CONFIG_PREEMPT
5174	/* Currently, only non dynamic ops can have a trampoline */
5175	if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
5176		return;
5177#endif
5178
5179	arch_ftrace_update_trampoline(ops);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5180}
5181
 
 
 
 
 
 
5182#else
5183
5184static struct ftrace_ops global_ops = {
5185	.func			= ftrace_stub,
5186	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
5187				  FTRACE_OPS_FL_INITIALIZED |
5188				  FTRACE_OPS_FL_PID,
5189};
5190
5191static int __init ftrace_nodyn_init(void)
5192{
5193	ftrace_enabled = 1;
5194	return 0;
5195}
5196core_initcall(ftrace_nodyn_init);
5197
5198static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
5199static inline void ftrace_startup_enable(int command) { }
5200static inline void ftrace_startup_all(int command) { }
5201/* Keep as macros so we do not need to define the commands */
5202# define ftrace_startup(ops, command)					\
5203	({								\
5204		int ___ret = __register_ftrace_function(ops);		\
5205		if (!___ret)						\
5206			(ops)->flags |= FTRACE_OPS_FL_ENABLED;		\
5207		___ret;							\
5208	})
5209# define ftrace_shutdown(ops, command)					\
5210	({								\
5211		int ___ret = __unregister_ftrace_function(ops);		\
5212		if (!___ret)						\
5213			(ops)->flags &= ~FTRACE_OPS_FL_ENABLED;		\
5214		___ret;							\
5215	})
5216
5217# define ftrace_startup_sysctl()	do { } while (0)
5218# define ftrace_shutdown_sysctl()	do { } while (0)
5219
5220static inline int
5221ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
5222{
5223	return 1;
5224}
5225
5226static void ftrace_update_trampoline(struct ftrace_ops *ops)
5227{
5228}
5229
5230#endif /* CONFIG_DYNAMIC_FTRACE */
5231
5232__init void ftrace_init_global_array_ops(struct trace_array *tr)
5233{
5234	tr->ops = &global_ops;
5235	tr->ops->private = tr;
 
5236}
5237
5238void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
5239{
5240	/* If we filter on pids, update to use the pid function */
5241	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
5242		if (WARN_ON(tr->ops->func != ftrace_stub))
5243			printk("ftrace ops had %pS for function\n",
5244			       tr->ops->func);
5245	}
5246	tr->ops->func = func;
5247	tr->ops->private = tr;
5248}
5249
5250void ftrace_reset_array_ops(struct trace_array *tr)
5251{
5252	tr->ops->func = ftrace_stub;
5253}
5254
5255static inline void
5256__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5257		       struct ftrace_ops *ignored, struct pt_regs *regs)
5258{
 
5259	struct ftrace_ops *op;
5260	int bit;
5261
5262	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
5263	if (bit < 0)
5264		return;
5265
5266	/*
5267	 * Some of the ops may be dynamically allocated,
5268	 * they must be freed after a synchronize_sched().
5269	 */
5270	preempt_disable_notrace();
5271
5272	do_for_each_ftrace_op(op, ftrace_ops_list) {
 
 
 
5273		/*
5274		 * Check the following for each ops before calling their func:
5275		 *  if RCU flag is set, then rcu_is_watching() must be true
5276		 *  if PER_CPU is set, then ftrace_function_local_disable()
5277		 *                          must be false
5278		 *  Otherwise test if the ip matches the ops filter
5279		 *
5280		 * If any of the above fails then the op->func() is not executed.
5281		 */
5282		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
5283		    (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
5284		     !ftrace_function_local_disabled(op)) &&
5285		    ftrace_ops_test(op, ip, regs)) {
5286		    
5287			if (FTRACE_WARN_ON(!op->func)) {
5288				pr_warn("op=%p %pS\n", op, op);
5289				goto out;
5290			}
5291			op->func(ip, parent_ip, op, regs);
5292		}
5293	} while_for_each_ftrace_op(op);
5294out:
5295	preempt_enable_notrace();
5296	trace_clear_recursion(bit);
5297}
5298
5299/*
5300 * Some archs only support passing ip and parent_ip. Even though
5301 * the list function ignores the op parameter, we do not want any
5302 * C side effects, where a function is called without the caller
5303 * sending a third parameter.
5304 * Archs are to support both the regs and ftrace_ops at the same time.
5305 * If they support ftrace_ops, it is assumed they support regs.
5306 * If call backs want to use regs, they must either check for regs
5307 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
5308 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
5309 * An architecture can pass partial regs with ftrace_ops and still
5310 * set the ARCH_SUPPORTS_FTRACE_OPS.
5311 */
5312#if ARCH_SUPPORTS_FTRACE_OPS
5313static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5314				 struct ftrace_ops *op, struct pt_regs *regs)
5315{
5316	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
5317}
 
5318#else
5319static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
5320{
5321	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
5322}
 
5323#endif
5324
5325/*
5326 * If there's only one function registered but it does not support
5327 * recursion, needs RCU protection and/or requires per cpu handling, then
5328 * this function will be called by the mcount trampoline.
5329 */
5330static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
5331				   struct ftrace_ops *op, struct pt_regs *regs)
5332{
5333	int bit;
5334
5335	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
5336		return;
5337
5338	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
5339	if (bit < 0)
5340		return;
5341
5342	preempt_disable_notrace();
5343
5344	if (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
5345	    !ftrace_function_local_disabled(op)) {
5346		op->func(ip, parent_ip, op, regs);
5347	}
5348
5349	preempt_enable_notrace();
5350	trace_clear_recursion(bit);
5351}
 
5352
5353/**
5354 * ftrace_ops_get_func - get the function a trampoline should call
5355 * @ops: the ops to get the function for
5356 *
5357 * Normally the mcount trampoline will call the ops->func, but there
5358 * are times that it should not. For example, if the ops does not
5359 * have its own recursion protection, then it should call the
5360 * ftrace_ops_recurs_func() instead.
5361 *
5362 * Returns the function that the trampoline should call for @ops.
5363 */
5364ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
5365{
5366	/*
5367	 * If the function does not handle recursion, needs to be RCU safe,
5368	 * or does per cpu logic, then we need to call the assist handler.
5369	 */
5370	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
5371	    ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU))
5372		return ftrace_ops_assist_func;
5373
5374	return ops->func;
5375}
5376
5377static void
5378ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
5379		    struct task_struct *prev, struct task_struct *next)
5380{
5381	struct trace_array *tr = data;
5382	struct trace_pid_list *pid_list;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5383
5384	pid_list = rcu_dereference_sched(tr->function_pids);
 
5385
5386	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
5387		       trace_ignore_this_task(pid_list, next));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5388}
5389
5390static void clear_ftrace_pids(struct trace_array *tr)
5391{
5392	struct trace_pid_list *pid_list;
 
5393	int cpu;
5394
5395	pid_list = rcu_dereference_protected(tr->function_pids,
5396					     lockdep_is_held(&ftrace_lock));
5397	if (!pid_list)
 
 
 
 
5398		return;
5399
5400	unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
 
 
 
 
 
5401
5402	for_each_possible_cpu(cpu)
5403		per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
5404
5405	rcu_assign_pointer(tr->function_pids, NULL);
 
5406
5407	/* Wait till all users are no longer using pid filtering */
5408	synchronize_sched();
 
 
 
5409
5410	trace_free_pid_list(pid_list);
 
 
 
 
 
 
 
 
 
 
5411}
5412
5413static void ftrace_pid_reset(struct trace_array *tr)
5414{
5415	mutex_lock(&ftrace_lock);
5416	clear_ftrace_pids(tr);
5417
5418	ftrace_update_pid_func();
5419	ftrace_startup_all(0);
5420
5421	mutex_unlock(&ftrace_lock);
5422}
5423
5424/* Greater than any max PID */
5425#define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
5426
5427static void *fpid_start(struct seq_file *m, loff_t *pos)
5428	__acquires(RCU)
5429{
5430	struct trace_pid_list *pid_list;
5431	struct trace_array *tr = m->private;
5432
5433	mutex_lock(&ftrace_lock);
5434	rcu_read_lock_sched();
5435
5436	pid_list = rcu_dereference_sched(tr->function_pids);
5437
5438	if (!pid_list)
5439		return !(*pos) ? FTRACE_NO_PIDS : NULL;
5440
5441	return trace_pid_start(pid_list, pos);
5442}
5443
5444static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
5445{
5446	struct trace_array *tr = m->private;
5447	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
5448
5449	if (v == FTRACE_NO_PIDS)
 
5450		return NULL;
5451
5452	return trace_pid_next(pid_list, v, pos);
5453}
5454
5455static void fpid_stop(struct seq_file *m, void *p)
5456	__releases(RCU)
5457{
5458	rcu_read_unlock_sched();
5459	mutex_unlock(&ftrace_lock);
5460}
5461
5462static int fpid_show(struct seq_file *m, void *v)
5463{
5464	if (v == FTRACE_NO_PIDS) {
5465		seq_puts(m, "no pid\n");
5466		return 0;
5467	}
5468
5469	return trace_pid_show(m, v);
5470}
5471
5472static const struct seq_operations ftrace_pid_sops = {
5473	.start = fpid_start,
5474	.next = fpid_next,
5475	.stop = fpid_stop,
5476	.show = fpid_show,
5477};
5478
5479static int
5480ftrace_pid_open(struct inode *inode, struct file *file)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5481{
 
5482	struct trace_array *tr = inode->i_private;
5483	struct seq_file *m;
5484	int ret = 0;
5485
5486	if (trace_array_get(tr) < 0)
5487		return -ENODEV;
 
5488
5489	if ((file->f_mode & FMODE_WRITE) &&
5490	    (file->f_flags & O_TRUNC))
5491		ftrace_pid_reset(tr);
5492
5493	ret = seq_open(file, &ftrace_pid_sops);
 
 
 
 
 
 
 
 
 
 
 
 
 
5494	if (ret < 0) {
5495		trace_array_put(tr);
5496	} else {
5497		m = file->private_data;
5498		/* copy tr over to seq ops */
5499		m->private = tr;
5500	}
5501
5502	return ret;
5503}
5504
 
 
 
 
 
 
 
 
 
 
 
 
5505static void ignore_task_cpu(void *data)
5506{
5507	struct trace_array *tr = data;
5508	struct trace_pid_list *pid_list;
 
5509
5510	/*
5511	 * This function is called by on_each_cpu() while the
5512	 * event_mutex is held.
5513	 */
5514	pid_list = rcu_dereference_protected(tr->function_pids,
5515					     mutex_is_locked(&ftrace_lock));
 
 
5516
5517	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
5518		       trace_ignore_this_task(pid_list, current));
 
 
 
 
5519}
5520
5521static ssize_t
5522ftrace_pid_write(struct file *filp, const char __user *ubuf,
5523		   size_t cnt, loff_t *ppos)
5524{
5525	struct seq_file *m = filp->private_data;
5526	struct trace_array *tr = m->private;
5527	struct trace_pid_list *filtered_pids = NULL;
 
5528	struct trace_pid_list *pid_list;
5529	ssize_t ret;
5530
5531	if (!cnt)
5532		return 0;
5533
5534	mutex_lock(&ftrace_lock);
5535
5536	filtered_pids = rcu_dereference_protected(tr->function_pids,
 
 
5537					     lockdep_is_held(&ftrace_lock));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5538
5539	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
5540	if (ret < 0)
5541		goto out;
5542
5543	rcu_assign_pointer(tr->function_pids, pid_list);
 
 
 
 
 
 
 
 
5544
5545	if (filtered_pids) {
5546		synchronize_sched();
5547		trace_free_pid_list(filtered_pids);
5548	} else if (pid_list) {
5549		/* Register a probe to set whether to ignore the tracing of a task */
5550		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
5551	}
5552
5553	/*
5554	 * Ignoring of pids is done at task switch. But we have to
5555	 * check for those tasks that are currently running.
5556	 * Always do this in case a pid was appended or removed.
5557	 */
5558	on_each_cpu(ignore_task_cpu, tr, 1);
5559
5560	ftrace_update_pid_func();
5561	ftrace_startup_all(0);
5562 out:
5563	mutex_unlock(&ftrace_lock);
5564
5565	if (ret > 0)
5566		*ppos += ret;
5567
5568	return ret;
5569}
5570
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5571static int
5572ftrace_pid_release(struct inode *inode, struct file *file)
5573{
5574	struct trace_array *tr = inode->i_private;
5575
5576	trace_array_put(tr);
5577
5578	return seq_release(inode, file);
5579}
5580
5581static const struct file_operations ftrace_pid_fops = {
5582	.open		= ftrace_pid_open,
5583	.write		= ftrace_pid_write,
5584	.read		= seq_read,
5585	.llseek		= tracing_lseek,
5586	.release	= ftrace_pid_release,
5587};
5588
 
 
 
 
 
 
 
 
5589void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
5590{
5591	trace_create_file("set_ftrace_pid", 0644, d_tracer,
5592			    tr, &ftrace_pid_fops);
 
 
5593}
5594
5595void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
5596					 struct dentry *d_tracer)
5597{
5598	/* Only the top level directory has the dyn_tracefs and profile */
5599	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
5600
5601	ftrace_init_dyn_tracefs(d_tracer);
5602	ftrace_profile_tracefs(d_tracer);
5603}
5604
5605/**
5606 * ftrace_kill - kill ftrace
5607 *
5608 * This function should be used by panic code. It stops ftrace
5609 * but in a not so nice way. If you need to simply kill ftrace
5610 * from a non-atomic section, use ftrace_kill.
5611 */
5612void ftrace_kill(void)
5613{
5614	ftrace_disabled = 1;
5615	ftrace_enabled = 0;
5616	clear_ftrace_function();
5617}
5618
5619/**
5620 * Test if ftrace is dead or not.
5621 */
5622int ftrace_is_dead(void)
5623{
5624	return ftrace_disabled;
5625}
5626
5627/**
5628 * register_ftrace_function - register a function for profiling
5629 * @ops - ops structure that holds the function for profiling.
5630 *
5631 * Register a function to be called by all functions in the
5632 * kernel.
5633 *
5634 * Note: @ops->func and all the functions it calls must be labeled
5635 *       with "notrace", otherwise it will go into a
5636 *       recursive loop.
5637 */
5638int register_ftrace_function(struct ftrace_ops *ops)
5639{
5640	int ret = -1;
5641
5642	ftrace_ops_init(ops);
5643
5644	mutex_lock(&ftrace_lock);
5645
5646	ret = ftrace_startup(ops, 0);
5647
5648	mutex_unlock(&ftrace_lock);
5649
5650	return ret;
5651}
5652EXPORT_SYMBOL_GPL(register_ftrace_function);
5653
5654/**
5655 * unregister_ftrace_function - unregister a function for profiling.
5656 * @ops - ops structure that holds the function to unregister
5657 *
5658 * Unregister a function that was added to be called by ftrace profiling.
5659 */
5660int unregister_ftrace_function(struct ftrace_ops *ops)
5661{
5662	int ret;
5663
5664	mutex_lock(&ftrace_lock);
5665	ret = ftrace_shutdown(ops, 0);
5666	mutex_unlock(&ftrace_lock);
5667
5668	return ret;
5669}
5670EXPORT_SYMBOL_GPL(unregister_ftrace_function);
5671
 
 
 
 
 
 
 
 
 
 
 
 
5672int
5673ftrace_enable_sysctl(struct ctl_table *table, int write,
5674		     void __user *buffer, size_t *lenp,
5675		     loff_t *ppos)
5676{
5677	int ret = -ENODEV;
5678
5679	mutex_lock(&ftrace_lock);
5680
5681	if (unlikely(ftrace_disabled))
5682		goto out;
5683
5684	ret = proc_dointvec(table, write, buffer, lenp, ppos);
5685
5686	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
5687		goto out;
5688
5689	last_ftrace_enabled = !!ftrace_enabled;
5690
5691	if (ftrace_enabled) {
5692
5693		/* we are starting ftrace again */
5694		if (ftrace_ops_list != &ftrace_list_end)
 
5695			update_ftrace_function();
5696
5697		ftrace_startup_sysctl();
5698
5699	} else {
 
 
 
 
 
 
5700		/* stopping ftrace calls (just send to ftrace_stub) */
5701		ftrace_trace_function = ftrace_stub;
5702
5703		ftrace_shutdown_sysctl();
5704	}
5705
 
5706 out:
5707	mutex_unlock(&ftrace_lock);
5708	return ret;
5709}
5710
5711#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5712
5713static struct ftrace_ops graph_ops = {
5714	.func			= ftrace_stub,
5715	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
5716				   FTRACE_OPS_FL_INITIALIZED |
5717				   FTRACE_OPS_FL_PID |
5718				   FTRACE_OPS_FL_STUB,
5719#ifdef FTRACE_GRAPH_TRAMP_ADDR
5720	.trampoline		= FTRACE_GRAPH_TRAMP_ADDR,
5721	/* trampoline_size is only needed for dynamically allocated tramps */
5722#endif
5723	ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
5724};
5725
5726void ftrace_graph_sleep_time_control(bool enable)
5727{
5728	fgraph_sleep_time = enable;
5729}
5730
5731void ftrace_graph_graph_time_control(bool enable)
5732{
5733	fgraph_graph_time = enable;
5734}
5735
5736int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
5737{
5738	return 0;
5739}
5740
5741/* The callbacks that hook a function */
5742trace_func_graph_ret_t ftrace_graph_return =
5743			(trace_func_graph_ret_t)ftrace_stub;
5744trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
5745static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
5746
5747/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
5748static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
5749{
5750	int i;
5751	int ret = 0;
5752	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
5753	struct task_struct *g, *t;
5754
5755	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
5756		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
5757					* sizeof(struct ftrace_ret_stack),
5758					GFP_KERNEL);
5759		if (!ret_stack_list[i]) {
5760			start = 0;
5761			end = i;
5762			ret = -ENOMEM;
5763			goto free;
5764		}
5765	}
5766
5767	read_lock(&tasklist_lock);
5768	do_each_thread(g, t) {
5769		if (start == end) {
5770			ret = -EAGAIN;
5771			goto unlock;
5772		}
5773
5774		if (t->ret_stack == NULL) {
5775			atomic_set(&t->tracing_graph_pause, 0);
5776			atomic_set(&t->trace_overrun, 0);
5777			t->curr_ret_stack = -1;
5778			/* Make sure the tasks see the -1 first: */
5779			smp_wmb();
5780			t->ret_stack = ret_stack_list[start++];
5781		}
5782	} while_each_thread(g, t);
5783
5784unlock:
5785	read_unlock(&tasklist_lock);
5786free:
5787	for (i = start; i < end; i++)
5788		kfree(ret_stack_list[i]);
5789	return ret;
5790}
5791
5792static void
5793ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
5794			struct task_struct *prev, struct task_struct *next)
5795{
5796	unsigned long long timestamp;
5797	int index;
5798
5799	/*
5800	 * Does the user want to count the time a function was asleep.
5801	 * If so, do not update the time stamps.
5802	 */
5803	if (fgraph_sleep_time)
5804		return;
5805
5806	timestamp = trace_clock_local();
5807
5808	prev->ftrace_timestamp = timestamp;
5809
5810	/* only process tasks that we timestamped */
5811	if (!next->ftrace_timestamp)
5812		return;
5813
5814	/*
5815	 * Update all the counters in next to make up for the
5816	 * time next was sleeping.
5817	 */
5818	timestamp -= next->ftrace_timestamp;
5819
5820	for (index = next->curr_ret_stack; index >= 0; index--)
5821		next->ret_stack[index].calltime += timestamp;
5822}
5823
5824/* Allocate a return stack for each task */
5825static int start_graph_tracing(void)
5826{
5827	struct ftrace_ret_stack **ret_stack_list;
5828	int ret, cpu;
5829
5830	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
5831				sizeof(struct ftrace_ret_stack *),
5832				GFP_KERNEL);
5833
5834	if (!ret_stack_list)
5835		return -ENOMEM;
5836
5837	/* The cpu_boot init_task->ret_stack will never be freed */
5838	for_each_online_cpu(cpu) {
5839		if (!idle_task(cpu)->ret_stack)
5840			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
5841	}
5842
5843	do {
5844		ret = alloc_retstack_tasklist(ret_stack_list);
5845	} while (ret == -EAGAIN);
5846
5847	if (!ret) {
5848		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
5849		if (ret)
5850			pr_info("ftrace_graph: Couldn't activate tracepoint"
5851				" probe to kernel_sched_switch\n");
5852	}
5853
5854	kfree(ret_stack_list);
5855	return ret;
5856}
5857
5858/*
5859 * Hibernation protection.
5860 * The state of the current task is too much unstable during
5861 * suspend/restore to disk. We want to protect against that.
5862 */
5863static int
5864ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
5865							void *unused)
5866{
5867	switch (state) {
5868	case PM_HIBERNATION_PREPARE:
5869		pause_graph_tracing();
5870		break;
5871
5872	case PM_POST_HIBERNATION:
5873		unpause_graph_tracing();
5874		break;
5875	}
5876	return NOTIFY_DONE;
5877}
5878
5879static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
5880{
5881	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
5882		return 0;
5883	return __ftrace_graph_entry(trace);
5884}
5885
5886/*
5887 * The function graph tracer should only trace the functions defined
5888 * by set_ftrace_filter and set_ftrace_notrace. If another function
5889 * tracer ops is registered, the graph tracer requires testing the
5890 * function against the global ops, and not just trace any function
5891 * that any ftrace_ops registered.
5892 */
5893static void update_function_graph_func(void)
5894{
5895	struct ftrace_ops *op;
5896	bool do_test = false;
5897
5898	/*
5899	 * The graph and global ops share the same set of functions
5900	 * to test. If any other ops is on the list, then
5901	 * the graph tracing needs to test if its the function
5902	 * it should call.
5903	 */
5904	do_for_each_ftrace_op(op, ftrace_ops_list) {
5905		if (op != &global_ops && op != &graph_ops &&
5906		    op != &ftrace_list_end) {
5907			do_test = true;
5908			/* in double loop, break out with goto */
5909			goto out;
5910		}
5911	} while_for_each_ftrace_op(op);
5912 out:
5913	if (do_test)
5914		ftrace_graph_entry = ftrace_graph_entry_test;
5915	else
5916		ftrace_graph_entry = __ftrace_graph_entry;
5917}
5918
5919static struct notifier_block ftrace_suspend_notifier = {
5920	.notifier_call = ftrace_suspend_notifier_call,
5921};
5922
5923int register_ftrace_graph(trace_func_graph_ret_t retfunc,
5924			trace_func_graph_ent_t entryfunc)
5925{
5926	int ret = 0;
5927
5928	mutex_lock(&ftrace_lock);
5929
5930	/* we currently allow only one tracer registered at a time */
5931	if (ftrace_graph_active) {
5932		ret = -EBUSY;
5933		goto out;
5934	}
5935
5936	register_pm_notifier(&ftrace_suspend_notifier);
5937
5938	ftrace_graph_active++;
5939	ret = start_graph_tracing();
5940	if (ret) {
5941		ftrace_graph_active--;
5942		goto out;
5943	}
5944
5945	ftrace_graph_return = retfunc;
5946
5947	/*
5948	 * Update the indirect function to the entryfunc, and the
5949	 * function that gets called to the entry_test first. Then
5950	 * call the update fgraph entry function to determine if
5951	 * the entryfunc should be called directly or not.
5952	 */
5953	__ftrace_graph_entry = entryfunc;
5954	ftrace_graph_entry = ftrace_graph_entry_test;
5955	update_function_graph_func();
5956
5957	ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
5958out:
5959	mutex_unlock(&ftrace_lock);
5960	return ret;
5961}
5962
5963void unregister_ftrace_graph(void)
5964{
5965	mutex_lock(&ftrace_lock);
5966
5967	if (unlikely(!ftrace_graph_active))
5968		goto out;
5969
5970	ftrace_graph_active--;
5971	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
5972	ftrace_graph_entry = ftrace_graph_entry_stub;
5973	__ftrace_graph_entry = ftrace_graph_entry_stub;
5974	ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
5975	unregister_pm_notifier(&ftrace_suspend_notifier);
5976	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
5977
5978#ifdef CONFIG_DYNAMIC_FTRACE
5979	/*
5980	 * Function graph does not allocate the trampoline, but
5981	 * other global_ops do. We need to reset the ALLOC_TRAMP flag
5982	 * if one was used.
5983	 */
5984	global_ops.trampoline = save_global_trampoline;
5985	if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP)
5986		global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
5987#endif
5988
5989 out:
5990	mutex_unlock(&ftrace_lock);
5991}
5992
5993static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
5994
5995static void
5996graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
5997{
5998	atomic_set(&t->tracing_graph_pause, 0);
5999	atomic_set(&t->trace_overrun, 0);
6000	t->ftrace_timestamp = 0;
6001	/* make curr_ret_stack visible before we add the ret_stack */
6002	smp_wmb();
6003	t->ret_stack = ret_stack;
6004}
6005
6006/*
6007 * Allocate a return stack for the idle task. May be the first
6008 * time through, or it may be done by CPU hotplug online.
6009 */
6010void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
6011{
6012	t->curr_ret_stack = -1;
6013	/*
6014	 * The idle task has no parent, it either has its own
6015	 * stack or no stack at all.
6016	 */
6017	if (t->ret_stack)
6018		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
6019
6020	if (ftrace_graph_active) {
6021		struct ftrace_ret_stack *ret_stack;
6022
6023		ret_stack = per_cpu(idle_ret_stack, cpu);
6024		if (!ret_stack) {
6025			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
6026					    * sizeof(struct ftrace_ret_stack),
6027					    GFP_KERNEL);
6028			if (!ret_stack)
6029				return;
6030			per_cpu(idle_ret_stack, cpu) = ret_stack;
6031		}
6032		graph_init_task(t, ret_stack);
6033	}
6034}
6035
6036/* Allocate a return stack for newly created task */
6037void ftrace_graph_init_task(struct task_struct *t)
6038{
6039	/* Make sure we do not use the parent ret_stack */
6040	t->ret_stack = NULL;
6041	t->curr_ret_stack = -1;
6042
6043	if (ftrace_graph_active) {
6044		struct ftrace_ret_stack *ret_stack;
6045
6046		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
6047				* sizeof(struct ftrace_ret_stack),
6048				GFP_KERNEL);
6049		if (!ret_stack)
6050			return;
6051		graph_init_task(t, ret_stack);
6052	}
6053}
6054
6055void ftrace_graph_exit_task(struct task_struct *t)
6056{
6057	struct ftrace_ret_stack	*ret_stack = t->ret_stack;
6058
6059	t->ret_stack = NULL;
6060	/* NULL must become visible to IRQs before we free it: */
6061	barrier();
6062
6063	kfree(ret_stack);
6064}
6065#endif