1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Kprobes-based tracing events
   4 *
   5 * Created by Masami Hiramatsu <mhiramat@redhat.com>
   6 *
 
 
 
 
 
 
 
 
 
 
 
 
   7 */
   8#define pr_fmt(fmt)	"trace_kprobe: " fmt
   9
  10#include <linux/bpf-cgroup.h>
  11#include <linux/security.h>
  12#include <linux/module.h>
  13#include <linux/uaccess.h>
  14#include <linux/rculist.h>
  15#include <linux/error-injection.h>
  16
  17#include <asm/setup.h>  /* for COMMAND_LINE_SIZE */
  18
  19#include "trace_dynevent.h"
  20#include "trace_kprobe_selftest.h"
  21#include "trace_probe.h"
  22#include "trace_probe_tmpl.h"
  23#include "trace_probe_kernel.h"
  24
 
 
 
 
 
 
 
 
 
 
  25#define KPROBE_EVENT_SYSTEM "kprobes"
  26#define KRETPROBE_MAXACTIVE_MAX 4096
  27
  28/* Kprobe early definition from command line */
  29static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata;
 
 
 
 
 
 
 
 
 
 
 
 
 
  30
  31static int __init set_kprobe_boot_events(char *str)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  32{
  33	strscpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE);
  34	disable_tracing_selftest("running kprobe events");
  35
  36	return 1;
  37}
  38__setup("kprobe_event=", set_kprobe_boot_events);
  39
  40static int trace_kprobe_create(const char *raw_command);
  41static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev);
  42static int trace_kprobe_release(struct dyn_event *ev);
  43static bool trace_kprobe_is_busy(struct dyn_event *ev);
  44static bool trace_kprobe_match(const char *system, const char *event,
  45			int argc, const char **argv, struct dyn_event *ev);
  46
  47static struct dyn_event_operations trace_kprobe_ops = {
  48	.create = trace_kprobe_create,
  49	.show = trace_kprobe_show,
  50	.is_busy = trace_kprobe_is_busy,
  51	.free = trace_kprobe_release,
  52	.match = trace_kprobe_match,
  53};
  54
  55/*
  56 * Kprobe event core functions
 
 
  57 */
  58struct trace_kprobe {
  59	struct dyn_event	devent;
  60	struct kretprobe	rp;	/* Use rp.kp for kprobe use */
  61	unsigned long __percpu *nhit;
  62	const char		*symbol;	/* symbol name */
  63	struct trace_probe	tp;
  64};
  65
  66static bool is_trace_kprobe(struct dyn_event *ev)
 
 
 
 
 
 
 
  67{
  68	return ev->ops == &trace_kprobe_ops;
 
 
 
 
 
 
  69}
 
  70
  71static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev)
 
 
 
 
 
 
 
 
 
  72{
  73	return container_of(ev, struct trace_kprobe, devent);
  74}
  75
  76/**
  77 * for_each_trace_kprobe - iterate over the trace_kprobe list
  78 * @pos:	the struct trace_kprobe * for each entry
  79 * @dpos:	the struct dyn_event * to use as a loop cursor
  80 */
  81#define for_each_trace_kprobe(pos, dpos)	\
  82	for_each_dyn_event(dpos)		\
  83		if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos)))
  84
  85static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  86{
  87	return tk->rp.handler != NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  88}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  89
  90static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
  91{
  92	return tk->symbol ? tk->symbol : "unknown";
 
  93}
  94
  95static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
 
 
 
 
 
 
 
  96{
  97	return tk->rp.kp.offset;
 
 
 
  98}
  99
 100static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
 101{
 102	return kprobe_gone(&tk->rp.kp);
 
 103}
 104
 105static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
 106						 struct module *mod)
 107{
 108	int len = strlen(module_name(mod));
 109	const char *name = trace_kprobe_symbol(tk);
 
 
 
 
 
 110
 111	return strncmp(module_name(mod), name, len) == 0 && name[len] == ':';
 
 
 
 
 
 
 
 
 112}
 113
 114#ifdef CONFIG_MODULES
 115static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
 116{
 117	char *p;
 118	bool ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 119
 120	if (!tk->symbol)
 121		return false;
 122	p = strchr(tk->symbol, ':');
 123	if (!p)
 124		return true;
 125	*p = '\0';
 126	rcu_read_lock_sched();
 127	ret = !!find_module(tk->symbol);
 128	rcu_read_unlock_sched();
 129	*p = ':';
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 130
 131	return ret;
 132}
 133#else
 134static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 135{
 136	return false;
 
 
 
 
 
 
 
 137}
 138#endif
 139
 140static bool trace_kprobe_is_busy(struct dyn_event *ev)
 
 141{
 142	struct trace_kprobe *tk = to_trace_kprobe(ev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 143
 144	return trace_probe_is_enabled(&tk->tp);
 145}
 146
 147static bool trace_kprobe_match_command_head(struct trace_kprobe *tk,
 148					    int argc, const char **argv)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 149{
 150	char buf[MAX_ARGSTR_LEN + 1];
 151
 152	if (!argc)
 153		return true;
 154
 155	if (!tk->symbol)
 156		snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr);
 157	else if (tk->rp.kp.offset)
 158		snprintf(buf, sizeof(buf), "%s+%u",
 159			 trace_kprobe_symbol(tk), tk->rp.kp.offset);
 160	else
 161		snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk));
 162	if (strcmp(buf, argv[0]))
 163		return false;
 164	argc--; argv++;
 
 
 
 
 
 
 
 
 
 
 
 
 165
 166	return trace_probe_match_command_args(&tk->tp, argc, argv);
 
 
 
 
 167}
 168
 169static bool trace_kprobe_match(const char *system, const char *event,
 170			int argc, const char **argv, struct dyn_event *ev)
 
 171{
 172	struct trace_kprobe *tk = to_trace_kprobe(ev);
 173
 174	return (event[0] == '\0' ||
 175		strcmp(trace_probe_name(&tk->tp), event) == 0) &&
 176	    (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) &&
 177	    trace_kprobe_match_command_head(tk, argc, argv);
 178}
 179
 180static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
 
 181{
 182	unsigned long nhit = 0;
 183	int cpu;
 184
 185	for_each_possible_cpu(cpu)
 186		nhit += *per_cpu_ptr(tk->nhit, cpu);
 
 
 
 187
 188	return nhit;
 
 189}
 190
 191static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 192{
 193	return !(list_empty(&tk->rp.kp.list) &&
 194		 hlist_unhashed(&tk->rp.kp.hlist));
 195}
 196
 197/* Return 0 if it fails to find the symbol address */
 198static nokprobe_inline
 199unsigned long trace_kprobe_address(struct trace_kprobe *tk)
 200{
 201	unsigned long addr;
 202
 203	if (tk->symbol) {
 204		addr = (unsigned long)
 205			kallsyms_lookup_name(trace_kprobe_symbol(tk));
 206		if (addr)
 207			addr += tk->rp.kp.offset;
 208	} else {
 209		addr = (unsigned long)tk->rp.kp.addr;
 210	}
 211	return addr;
 212}
 213
 214static nokprobe_inline struct trace_kprobe *
 215trace_kprobe_primary_from_call(struct trace_event_call *call)
 216{
 217	struct trace_probe *tp;
 218
 219	tp = trace_probe_primary_from_call(call);
 220	if (WARN_ON_ONCE(!tp))
 221		return NULL;
 222
 223	return container_of(tp, struct trace_kprobe, tp);
 
 
 224}
 225
 226bool trace_kprobe_on_func_entry(struct trace_event_call *call)
 227{
 228	struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
 
 229
 230	return tk ? (kprobe_on_func_entry(tk->rp.kp.addr,
 231			tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
 232			tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false;
 233}
 234
 235bool trace_kprobe_error_injectable(struct trace_event_call *call)
 
 236{
 237	struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
 
 
 
 238
 239	return tk ? within_error_injection_list(trace_kprobe_address(tk)) :
 240	       false;
 
 241}
 242
 243static int register_kprobe_event(struct trace_kprobe *tk);
 244static int unregister_kprobe_event(struct trace_kprobe *tk);
 
 
 
 245
 246static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
 247static int kretprobe_dispatcher(struct kretprobe_instance *ri,
 248				struct pt_regs *regs);
 249
 250static void free_trace_kprobe(struct trace_kprobe *tk)
 
 251{
 252	if (tk) {
 253		trace_probe_cleanup(&tk->tp);
 254		kfree(tk->symbol);
 255		free_percpu(tk->nhit);
 256		kfree(tk);
 257	}
 
 258}
 259
 260/*
 261 * Allocate new trace_probe and initialize it (including kprobes).
 262 */
 263static struct trace_kprobe *alloc_trace_kprobe(const char *group,
 264					     const char *event,
 265					     void *addr,
 266					     const char *symbol,
 267					     unsigned long offs,
 268					     int maxactive,
 269					     int nargs, bool is_return)
 270{
 271	struct trace_kprobe *tk;
 272	int ret = -ENOMEM;
 273
 274	tk = kzalloc(struct_size(tk, tp.args, nargs), GFP_KERNEL);
 275	if (!tk)
 276		return ERR_PTR(ret);
 277
 278	tk->nhit = alloc_percpu(unsigned long);
 279	if (!tk->nhit)
 280		goto error;
 281
 282	if (symbol) {
 283		tk->symbol = kstrdup(symbol, GFP_KERNEL);
 284		if (!tk->symbol)
 285			goto error;
 286		tk->rp.kp.symbol_name = tk->symbol;
 287		tk->rp.kp.offset = offs;
 288	} else
 289		tk->rp.kp.addr = addr;
 290
 291	if (is_return)
 292		tk->rp.handler = kretprobe_dispatcher;
 293	else
 294		tk->rp.kp.pre_handler = kprobe_dispatcher;
 295
 296	tk->rp.maxactive = maxactive;
 297	INIT_HLIST_NODE(&tk->rp.kp.hlist);
 298	INIT_LIST_HEAD(&tk->rp.kp.list);
 
 299
 300	ret = trace_probe_init(&tk->tp, event, group, false, nargs);
 301	if (ret < 0)
 
 302		goto error;
 303
 304	dyn_event_init(&tk->devent, &trace_kprobe_ops);
 305	return tk;
 
 
 
 
 
 
 
 
 
 306error:
 307	free_trace_kprobe(tk);
 
 
 308	return ERR_PTR(ret);
 309}
 310
 311static struct trace_kprobe *find_trace_kprobe(const char *event,
 312					      const char *group)
 313{
 314	struct dyn_event *pos;
 315	struct trace_kprobe *tk;
 
 
 
 
 
 316
 317	for_each_trace_kprobe(tk, pos)
 318		if (strcmp(trace_probe_name(&tk->tp), event) == 0 &&
 319		    strcmp(trace_probe_group_name(&tk->tp), group) == 0)
 320			return tk;
 321	return NULL;
 
 
 
 
 
 322}
 323
 324static inline int __enable_trace_kprobe(struct trace_kprobe *tk)
 325{
 326	int ret = 0;
 327
 328	if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) {
 329		if (trace_kprobe_is_return(tk))
 330			ret = enable_kretprobe(&tk->rp);
 331		else
 332			ret = enable_kprobe(&tk->rp.kp);
 333	}
 334
 335	return ret;
 
 
 
 336}
 337
 338static void __disable_trace_kprobe(struct trace_probe *tp)
 
 339{
 340	struct trace_kprobe *tk;
 341
 342	list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {
 343		if (!trace_kprobe_is_registered(tk))
 344			continue;
 345		if (trace_kprobe_is_return(tk))
 346			disable_kretprobe(&tk->rp);
 347		else
 348			disable_kprobe(&tk->rp.kp);
 349	}
 350}
 351
 352/*
 353 * Enable trace_probe
 354 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
 355 */
 356static int enable_trace_kprobe(struct trace_event_call *call,
 357				struct trace_event_file *file)
 358{
 359	struct trace_probe *tp;
 360	struct trace_kprobe *tk;
 361	bool enabled;
 362	int ret = 0;
 363
 364	tp = trace_probe_primary_from_call(call);
 365	if (WARN_ON_ONCE(!tp))
 366		return -ENODEV;
 367	enabled = trace_probe_is_enabled(tp);
 368
 369	/* This also changes "enabled" state */
 370	if (file) {
 371		ret = trace_probe_add_file(tp, file);
 372		if (ret)
 373			return ret;
 374	} else
 375		trace_probe_set_flag(tp, TP_FLAG_PROFILE);
 376
 377	if (enabled)
 378		return 0;
 379
 380	list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {
 381		if (trace_kprobe_has_gone(tk))
 382			continue;
 383		ret = __enable_trace_kprobe(tk);
 384		if (ret)
 385			break;
 386		enabled = true;
 387	}
 388
 389	if (ret) {
 390		/* Failed to enable one of them. Roll back all */
 391		if (enabled)
 392			__disable_trace_kprobe(tp);
 393		if (file)
 394			trace_probe_remove_file(tp, file);
 395		else
 396			trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
 397	}
 398
 399	return ret;
 400}
 401
 402/*
 403 * Disable trace_probe
 404 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
 405 */
 406static int disable_trace_kprobe(struct trace_event_call *call,
 407				struct trace_event_file *file)
 408{
 409	struct trace_probe *tp;
 410
 411	tp = trace_probe_primary_from_call(call);
 412	if (WARN_ON_ONCE(!tp))
 413		return -ENODEV;
 414
 415	if (file) {
 416		if (!trace_probe_get_file_link(tp, file))
 417			return -ENOENT;
 418		if (!trace_probe_has_single_file(tp))
 419			goto out;
 420		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
 421	} else
 422		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
 423
 424	if (!trace_probe_is_enabled(tp))
 425		__disable_trace_kprobe(tp);
 426
 427 out:
 428	if (file)
 429		/*
 430		 * Synchronization is done in below function. For perf event,
 431		 * file == NULL and perf_trace_event_unreg() calls
 432		 * tracepoint_synchronize_unregister() to ensure synchronize
 433		 * event. We don't need to care about it.
 434		 */
 435		trace_probe_remove_file(tp, file);
 436
 437	return 0;
 438}
 439
 440#if defined(CONFIG_DYNAMIC_FTRACE) && \
 441	!defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)
 442static bool __within_notrace_func(unsigned long addr)
 443{
 444	unsigned long offset, size;
 445
 446	if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset))
 447		return false;
 448
 449	/* Get the entry address of the target function */
 450	addr -= offset;
 451
 452	/*
 453	 * Since ftrace_location_range() does inclusive range check, we need
 454	 * to subtract 1 byte from the end address.
 455	 */
 456	return !ftrace_location_range(addr, addr + size - 1);
 457}
 458
 459static bool within_notrace_func(struct trace_kprobe *tk)
 460{
 461	unsigned long addr = trace_kprobe_address(tk);
 462	char symname[KSYM_NAME_LEN], *p;
 463
 464	if (!__within_notrace_func(addr))
 465		return false;
 466
 467	/* Check if the address is on a suffixed-symbol */
 468	if (!lookup_symbol_name(addr, symname)) {
 469		p = strchr(symname, '.');
 470		if (!p)
 471			return true;
 472		*p = '\0';
 473		addr = (unsigned long)kprobe_lookup_name(symname, 0);
 474		if (addr)
 475			return __within_notrace_func(addr);
 476	}
 477
 478	return true;
 479}
 480#else
 481#define within_notrace_func(tk)	(false)
 482#endif
 483
 484/* Internal register function - just handle k*probes and flags */
 485static int __register_trace_kprobe(struct trace_kprobe *tk)
 486{
 487	int i, ret;
 488
 489	ret = security_locked_down(LOCKDOWN_KPROBES);
 490	if (ret)
 491		return ret;
 492
 493	if (trace_kprobe_is_registered(tk))
 494		return -EINVAL;
 495
 496	if (within_notrace_func(tk)) {
 497		pr_warn("Could not probe notrace function %ps\n",
 498			(void *)trace_kprobe_address(tk));
 499		return -EINVAL;
 500	}
 501
 502	for (i = 0; i < tk->tp.nr_args; i++) {
 503		ret = traceprobe_update_arg(&tk->tp.args[i]);
 504		if (ret)
 505			return ret;
 506	}
 507
 508	/* Set/clear disabled flag according to tp->flag */
 509	if (trace_probe_is_enabled(&tk->tp))
 510		tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
 511	else
 512		tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
 513
 514	if (trace_kprobe_is_return(tk))
 515		ret = register_kretprobe(&tk->rp);
 516	else
 517		ret = register_kprobe(&tk->rp.kp);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 518
 519	return ret;
 520}
 521
 522/* Internal unregister function - just handle k*probes and flags */
 523static void __unregister_trace_kprobe(struct trace_kprobe *tk)
 524{
 525	if (trace_kprobe_is_registered(tk)) {
 526		if (trace_kprobe_is_return(tk))
 527			unregister_kretprobe(&tk->rp);
 528		else
 529			unregister_kprobe(&tk->rp.kp);
 530		/* Cleanup kprobe for reuse and mark it unregistered */
 531		INIT_HLIST_NODE(&tk->rp.kp.hlist);
 532		INIT_LIST_HEAD(&tk->rp.kp.list);
 533		if (tk->rp.kp.symbol_name)
 534			tk->rp.kp.addr = NULL;
 535	}
 536}
 537
 538/* Unregister a trace_probe and probe_event */
 539static int unregister_trace_kprobe(struct trace_kprobe *tk)
 540{
 541	/* If other probes are on the event, just unregister kprobe */
 542	if (trace_probe_has_sibling(&tk->tp))
 543		goto unreg;
 544
 545	/* Enabled event can not be unregistered */
 546	if (trace_probe_is_enabled(&tk->tp))
 547		return -EBUSY;
 548
 549	/* If there's a reference to the dynamic event */
 550	if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp)))
 551		return -EBUSY;
 552
 553	/* Will fail if probe is being used by ftrace or perf */
 554	if (unregister_kprobe_event(tk))
 555		return -EBUSY;
 556
 557unreg:
 558	__unregister_trace_kprobe(tk);
 559	dyn_event_remove(&tk->devent);
 560	trace_probe_unlink(&tk->tp);
 561
 562	return 0;
 563}
 564
 565static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig,
 566					 struct trace_kprobe *comp)
 567{
 568	struct trace_probe_event *tpe = orig->tp.event;
 569	int i;
 570
 571	list_for_each_entry(orig, &tpe->probes, tp.list) {
 572		if (strcmp(trace_kprobe_symbol(orig),
 573			   trace_kprobe_symbol(comp)) ||
 574		    trace_kprobe_offset(orig) != trace_kprobe_offset(comp))
 575			continue;
 576
 577		/*
 578		 * trace_probe_compare_arg_type() ensured that nr_args and
 579		 * each argument name and type are same. Let's compare comm.
 580		 */
 581		for (i = 0; i < orig->tp.nr_args; i++) {
 582			if (strcmp(orig->tp.args[i].comm,
 583				   comp->tp.args[i].comm))
 584				break;
 585		}
 586
 587		if (i == orig->tp.nr_args)
 588			return true;
 589	}
 590
 591	return false;
 592}
 593
 594static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to)
 595{
 596	int ret;
 597
 598	ret = trace_probe_compare_arg_type(&tk->tp, &to->tp);
 599	if (ret) {
 600		/* Note that argument starts index = 2 */
 601		trace_probe_log_set_index(ret + 1);
 602		trace_probe_log_err(0, DIFF_ARG_TYPE);
 603		return -EEXIST;
 604	}
 605	if (trace_kprobe_has_same_kprobe(to, tk)) {
 606		trace_probe_log_set_index(0);
 607		trace_probe_log_err(0, SAME_PROBE);
 608		return -EEXIST;
 609	}
 610
 611	/* Append to existing event */
 612	ret = trace_probe_append(&tk->tp, &to->tp);
 613	if (ret)
 614		return ret;
 615
 616	/* Register k*probe */
 617	ret = __register_trace_kprobe(tk);
 618	if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
 619		pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
 620		ret = 0;
 621	}
 622
 623	if (ret)
 624		trace_probe_unlink(&tk->tp);
 625	else
 626		dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));
 627
 628	return ret;
 629}
 630
 631/* Register a trace_probe and probe_event */
 632static int register_trace_kprobe(struct trace_kprobe *tk)
 633{
 634	struct trace_kprobe *old_tk;
 635	int ret;
 636
 637	mutex_lock(&event_mutex);
 638
 639	old_tk = find_trace_kprobe(trace_probe_name(&tk->tp),
 640				   trace_probe_group_name(&tk->tp));
 641	if (old_tk) {
 642		if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) {
 643			trace_probe_log_set_index(0);
 644			trace_probe_log_err(0, DIFF_PROBE_TYPE);
 645			ret = -EEXIST;
 646		} else {
 647			ret = append_trace_kprobe(tk, old_tk);
 648		}
 649		goto end;
 650	}
 651
 652	/* Register new event */
 653	ret = register_kprobe_event(tk);
 654	if (ret) {
 655		if (ret == -EEXIST) {
 656			trace_probe_log_set_index(0);
 657			trace_probe_log_err(0, EVENT_EXIST);
 658		} else
 659			pr_warn("Failed to register probe event(%d)\n", ret);
 660		goto end;
 661	}
 662
 663	/* Register k*probe */
 664	ret = __register_trace_kprobe(tk);
 665	if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
 666		pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
 667		ret = 0;
 668	}
 669
 670	if (ret < 0)
 671		unregister_kprobe_event(tk);
 672	else
 673		dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));
 674
 675end:
 676	mutex_unlock(&event_mutex);
 677	return ret;
 678}
 679
 680#ifdef CONFIG_MODULES
 681static int validate_module_probe_symbol(const char *modname, const char *symbol);
 682
 683static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk)
 684{
 685	const char *p;
 686	int ret = 0;
 687
 688	p = strchr(trace_kprobe_symbol(tk), ':');
 689	if (p)
 690		ret = validate_module_probe_symbol(module_name(mod), p + 1);
 691	if (!ret)
 692		ret = __register_trace_kprobe(tk);
 693	return ret;
 694}
 695
 696/* Module notifier call back, checking event on the module */
 697static int trace_kprobe_module_callback(struct notifier_block *nb,
 698				       unsigned long val, void *data)
 699{
 700	struct module *mod = data;
 701	struct dyn_event *pos;
 702	struct trace_kprobe *tk;
 703	int ret;
 704
 705	if (val != MODULE_STATE_COMING)
 706		return NOTIFY_DONE;
 707
 708	/* Update probes on coming module */
 709	mutex_lock(&event_mutex);
 710	for_each_trace_kprobe(tk, pos) {
 711		if (trace_kprobe_within_module(tk, mod)) {
 712			/* Don't need to check busy - this should have gone. */
 713			__unregister_trace_kprobe(tk);
 714			ret = register_module_trace_kprobe(mod, tk);
 715			if (ret)
 716				pr_warn("Failed to re-register probe %s on %s: %d\n",
 717					trace_probe_name(&tk->tp),
 718					module_name(mod), ret);
 719		}
 720	}
 721	mutex_unlock(&event_mutex);
 722
 723	return NOTIFY_DONE;
 724}
 725
 726static struct notifier_block trace_kprobe_module_nb = {
 727	.notifier_call = trace_kprobe_module_callback,
 728	.priority = 2	/* Invoked after kprobe and jump_label module callback */
 729};
 730static int trace_kprobe_register_module_notifier(void)
 731{
 732	return register_module_notifier(&trace_kprobe_module_nb);
 733}
 734#else
 735static int trace_kprobe_register_module_notifier(void)
 736{
 737	return 0;
 738}
 739#endif /* CONFIG_MODULES */
 740
 741static int count_symbols(void *data, unsigned long unused)
 
 742{
 743	unsigned int *count = data;
 
 744
 745	(*count)++;
 
 746
 
 
 
 
 
 
 
 
 
 747	return 0;
 748}
 749
 750struct sym_count_ctx {
 751	unsigned int count;
 752	const char *name;
 753};
 754
 755static int count_mod_symbols(void *data, const char *name, unsigned long unused)
 
 756{
 757	struct sym_count_ctx *ctx = data;
 758
 759	if (strcmp(name, ctx->name) == 0)
 760		ctx->count++;
 761
 762	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 763}
 764
 765static unsigned int number_of_same_symbols(const char *mod, const char *func_name)
 
 
 766{
 767	struct sym_count_ctx ctx = { .count = 0, .name = func_name };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 768
 769	if (!mod)
 770		kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);
 771
 772	module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx);
 
 
 
 
 
 
 
 773
 774	return ctx.count;
 775}
 776
 777static int validate_module_probe_symbol(const char *modname, const char *symbol)
 778{
 779	unsigned int count = number_of_same_symbols(modname, symbol);
 
 
 
 780
 781	if (count > 1) {
 782		/*
 783		 * Users should use ADDR to remove the ambiguity of
 784		 * using KSYM only.
 785		 */
 786		return -EADDRNOTAVAIL;
 787	} else if (count == 0) {
 788		/*
 789		 * We can return ENOENT earlier than when register the
 790		 * kprobe.
 791		 */
 792		return -ENOENT;
 793	}
 794	return 0;
 795}
 796
 797#ifdef CONFIG_MODULES
 798/* Return NULL if the module is not loaded or under unloading. */
 799static struct module *try_module_get_by_name(const char *name)
 800{
 801	struct module *mod;
 802
 803	rcu_read_lock_sched();
 804	mod = find_module(name);
 805	if (mod && !try_module_get(mod))
 806		mod = NULL;
 807	rcu_read_unlock_sched();
 808
 809	return mod;
 
 
 810}
 811#else
 812#define try_module_get_by_name(name)	(NULL)
 813#endif
 814
 815static int validate_probe_symbol(char *symbol)
 
 
 816{
 817	struct module *mod = NULL;
 818	char *modname = NULL, *p;
 819	int ret = 0;
 820
 821	p = strchr(symbol, ':');
 822	if (p) {
 823		modname = symbol;
 824		symbol = p + 1;
 825		*p = '\0';
 826		mod = try_module_get_by_name(modname);
 827		if (!mod)
 828			goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 829	}
 830
 831	ret = validate_module_probe_symbol(modname, symbol);
 832out:
 833	if (p)
 834		*p = ':';
 835	if (mod)
 836		module_put(mod);
 837	return ret;
 838}
 839
 840static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,
 841				      struct pt_regs *regs);
 
 
 
 
 
 
 
 
 
 
 
 842
 843static int __trace_kprobe_create(int argc, const char *argv[])
 844{
 845	/*
 846	 * Argument syntax:
 847	 *  - Add kprobe:
 848	 *      p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
 849	 *  - Add kretprobe:
 850	 *      r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS]
 851	 *    Or
 852	 *      p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS]
 853	 *
 854	 * Fetch args:
 855	 *  $retval	: fetch return value
 856	 *  $stack	: fetch stack address
 857	 *  $stackN	: fetch Nth of stack (N:0-)
 858	 *  $comm       : fetch current task comm
 859	 *  @ADDR	: fetch memory at ADDR (ADDR should be in kernel)
 860	 *  @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
 861	 *  %REG	: fetch register REG
 862	 * Dereferencing memory fetch:
 863	 *  +|-offs(ARG) : fetch memory at ARG +|- offs address.
 864	 * Alias name of args:
 865	 *  NAME=FETCHARG : set NAME as alias of FETCHARG.
 866	 * Type of args:
 867	 *  FETCHARG:TYPE : use TYPE instead of unsigned long.
 868	 */
 869	struct trace_kprobe *tk = NULL;
 870	int i, len, new_argc = 0, ret = 0;
 871	bool is_return = false;
 872	char *symbol = NULL, *tmp = NULL;
 873	const char **new_argv = NULL;
 874	const char *event = NULL, *group = KPROBE_EVENT_SYSTEM;
 875	enum probe_print_type ptype;
 876	int maxactive = 0;
 877	long offset = 0;
 878	void *addr = NULL;
 879	char buf[MAX_EVENT_NAME_LEN];
 880	char gbuf[MAX_EVENT_NAME_LEN];
 881	char abuf[MAX_BTF_ARGS_LEN];
 882	char *dbuf = NULL;
 883	struct traceprobe_parse_context ctx = { .flags = TPARG_FL_KERNEL };
 884
 885	switch (argv[0][0]) {
 886	case 'r':
 887		is_return = true;
 888		break;
 889	case 'p':
 890		break;
 891	default:
 892		return -ECANCELED;
 893	}
 894	if (argc < 2)
 895		return -ECANCELED;
 896
 897	trace_probe_log_init("trace_kprobe", argc, argv);
 898
 899	event = strchr(&argv[0][1], ':');
 900	if (event)
 901		event++;
 
 
 
 
 
 
 
 902
 903	if (isdigit(argv[0][1])) {
 904		if (!is_return) {
 905			trace_probe_log_err(1, BAD_MAXACT_TYPE);
 906			goto parse_error;
 
 
 
 
 
 
 907		}
 908		if (event)
 909			len = event - &argv[0][1] - 1;
 910		else
 911			len = strlen(&argv[0][1]);
 912		if (len > MAX_EVENT_NAME_LEN - 1) {
 913			trace_probe_log_err(1, BAD_MAXACT);
 914			goto parse_error;
 915		}
 916		memcpy(buf, &argv[0][1], len);
 917		buf[len] = '\0';
 918		ret = kstrtouint(buf, 0, &maxactive);
 919		if (ret || !maxactive) {
 920			trace_probe_log_err(1, BAD_MAXACT);
 921			goto parse_error;
 922		}
 923		/* kretprobes instances are iterated over via a list. The
 924		 * maximum should stay reasonable.
 925		 */
 926		if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
 927			trace_probe_log_err(1, MAXACT_TOO_BIG);
 928			goto parse_error;
 929		}
 930	}
 
 
 931
 932	/* try to parse an address. if that fails, try to read the
 933	 * input as a symbol. */
 934	if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
 935		trace_probe_log_set_index(1);
 936		/* Check whether uprobe event specified */
 937		if (strchr(argv[1], '/') && strchr(argv[1], ':')) {
 938			ret = -ECANCELED;
 939			goto error;
 940		}
 941		/* a symbol specified */
 942		symbol = kstrdup(argv[1], GFP_KERNEL);
 943		if (!symbol) {
 944			ret = -ENOMEM;
 945			goto error;
 
 946		}
 
 
 
 
 
 
 947
 948		tmp = strchr(symbol, '%');
 949		if (tmp) {
 950			if (!strcmp(tmp, "%return")) {
 951				*tmp = '\0';
 952				is_return = true;
 953			} else {
 954				trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX);
 955				goto parse_error;
 956			}
 957		}
 958
 959		/* TODO: support .init module functions */
 960		ret = traceprobe_split_symbol_offset(symbol, &offset);
 961		if (ret || offset < 0 || offset > UINT_MAX) {
 962			trace_probe_log_err(0, BAD_PROBE_ADDR);
 963			goto parse_error;
 964		}
 965		ret = validate_probe_symbol(symbol);
 
 
 
 
 966		if (ret) {
 967			if (ret == -EADDRNOTAVAIL)
 968				trace_probe_log_err(0, NON_UNIQ_SYMBOL);
 969			else
 970				trace_probe_log_err(0, BAD_PROBE_ADDR);
 971			goto parse_error;
 972		}
 973		if (is_return)
 974			ctx.flags |= TPARG_FL_RETURN;
 975		ret = kprobe_on_func_entry(NULL, symbol, offset);
 976		if (ret == 0 && !is_return)
 977			ctx.flags |= TPARG_FL_FENTRY;
 978		/* Defer the ENOENT case until register kprobe */
 979		if (ret == -EINVAL && is_return) {
 980			trace_probe_log_err(0, BAD_RETPROBE);
 981			goto parse_error;
 982		}
 983	}
 
 984
 985	trace_probe_log_set_index(0);
 986	if (event) {
 987		ret = traceprobe_parse_event_name(&event, &group, gbuf,
 988						  event - argv[0]);
 989		if (ret)
 990			goto parse_error;
 991	}
 992
 993	if (!event) {
 994		/* Make a new event name */
 995		if (symbol)
 996			snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
 997				 is_return ? 'r' : 'p', symbol, offset);
 998		else
 999			snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
1000				 is_return ? 'r' : 'p', addr);
1001		sanitize_event_name(buf);
1002		event = buf;
1003	}
1004
1005	argc -= 2; argv += 2;
1006	ctx.funcname = symbol;
1007	new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
1008					       abuf, MAX_BTF_ARGS_LEN, &ctx);
1009	if (IS_ERR(new_argv)) {
1010		ret = PTR_ERR(new_argv);
1011		new_argv = NULL;
1012		goto out;
1013	}
1014	if (new_argv) {
1015		argc = new_argc;
1016		argv = new_argv;
1017	}
1018	if (argc > MAX_TRACE_ARGS) {
1019		ret = -E2BIG;
1020		goto out;
1021	}
1022
1023	ret = traceprobe_expand_dentry_args(argc, argv, &dbuf);
1024	if (ret)
1025		goto out;
1026
1027	/* setup a probe */
1028	tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
1029				argc, is_return);
1030	if (IS_ERR(tk)) {
1031		ret = PTR_ERR(tk);
1032		/* This must return -ENOMEM, else there is a bug */
1033		WARN_ON_ONCE(ret != -ENOMEM);
1034		goto out;	/* We know tk is not allocated */
1035	}
1036
1037	/* parse arguments */
1038	for (i = 0; i < argc; i++) {
1039		trace_probe_log_set_index(i + 2);
1040		ctx.offset = 0;
1041		ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], &ctx);
1042		if (ret)
1043			goto error;	/* This can be -ENOMEM */
1044	}
1045	/* entry handler for kretprobe */
1046	if (is_return && tk->tp.entry_arg) {
1047		tk->rp.entry_handler = trace_kprobe_entry_handler;
1048		tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp);
1049	}
1050
1051	ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1052	ret = traceprobe_set_print_fmt(&tk->tp, ptype);
1053	if (ret < 0)
1054		goto error;
1055
1056	ret = register_trace_kprobe(tk);
1057	if (ret) {
1058		trace_probe_log_set_index(1);
1059		if (ret == -EILSEQ)
1060			trace_probe_log_err(0, BAD_INSN_BNDRY);
1061		else if (ret == -ENOENT)
1062			trace_probe_log_err(0, BAD_PROBE_ADDR);
1063		else if (ret != -ENOMEM && ret != -EEXIST)
1064			trace_probe_log_err(0, FAIL_REG_PROBE);
1065		goto error;
1066	}
1067
1068out:
1069	traceprobe_finish_parse(&ctx);
1070	trace_probe_log_clear();
1071	kfree(new_argv);
1072	kfree(symbol);
1073	kfree(dbuf);
1074	return ret;
1075
1076parse_error:
1077	ret = -EINVAL;
1078error:
1079	free_trace_kprobe(tk);
1080	goto out;
1081}
1082
1083static int trace_kprobe_create(const char *raw_command)
1084{
1085	return trace_probe_create(raw_command, __trace_kprobe_create);
1086}
1087
1088static int create_or_delete_trace_kprobe(const char *raw_command)
1089{
1090	int ret;
1091
1092	if (raw_command[0] == '-')
1093		return dyn_event_release(raw_command, &trace_kprobe_ops);
1094
1095	ret = trace_kprobe_create(raw_command);
1096	return ret == -ECANCELED ? -EINVAL : ret;
1097}
1098
1099static int trace_kprobe_run_command(struct dynevent_cmd *cmd)
1100{
1101	return create_or_delete_trace_kprobe(cmd->seq.buffer);
1102}
1103
1104/**
1105 * kprobe_event_cmd_init - Initialize a kprobe event command object
1106 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1107 * @buf: A pointer to the buffer used to build the command
1108 * @maxlen: The length of the buffer passed in @buf
1109 *
1110 * Initialize a synthetic event command object.  Use this before
1111 * calling any of the other kprobe_event functions.
1112 */
1113void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen)
1114{
1115	dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE,
1116			  trace_kprobe_run_command);
1117}
1118EXPORT_SYMBOL_GPL(kprobe_event_cmd_init);
1119
1120/**
1121 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list
1122 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1123 * @kretprobe: Is this a return probe?
1124 * @name: The name of the kprobe event
1125 * @loc: The location of the kprobe event
1126 * @...: Variable number of arg (pairs), one pair for each field
1127 *
1128 * NOTE: Users normally won't want to call this function directly, but
1129 * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically
1130 * adds a NULL to the end of the arg list.  If this function is used
1131 * directly, make sure the last arg in the variable arg list is NULL.
1132 *
1133 * Generate a kprobe event command to be executed by
1134 * kprobe_event_gen_cmd_end().  This function can be used to generate the
1135 * complete command or only the first part of it; in the latter case,
1136 * kprobe_event_add_fields() can be used to add more fields following this.
1137 *
1138 * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This
1139 * returns -EINVAL if @loc == NULL.
1140 *
1141 * Return: 0 if successful, error otherwise.
1142 */
1143int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe,
1144				 const char *name, const char *loc, ...)
1145{
1146	char buf[MAX_EVENT_NAME_LEN];
1147	struct dynevent_arg arg;
1148	va_list args;
1149	int ret;
1150
1151	if (cmd->type != DYNEVENT_TYPE_KPROBE)
1152		return -EINVAL;
1153
1154	if (!loc)
1155		return -EINVAL;
1156
1157	if (kretprobe)
1158		snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name);
1159	else
1160		snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name);
1161
1162	ret = dynevent_str_add(cmd, buf);
1163	if (ret)
1164		return ret;
1165
1166	dynevent_arg_init(&arg, 0);
1167	arg.str = loc;
1168	ret = dynevent_arg_add(cmd, &arg, NULL);
1169	if (ret)
1170		return ret;
1171
1172	va_start(args, loc);
1173	for (;;) {
1174		const char *field;
 
 
1175
1176		field = va_arg(args, const char *);
1177		if (!field)
1178			break;
 
 
 
1179
1180		if (++cmd->n_fields > MAX_TRACE_ARGS) {
 
 
1181			ret = -EINVAL;
1182			break;
1183		}
1184
1185		arg.str = field;
1186		ret = dynevent_arg_add(cmd, &arg, NULL);
1187		if (ret)
1188			break;
 
 
1189	}
1190	va_end(args);
1191
 
 
 
 
 
 
 
1192	return ret;
1193}
1194EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start);
1195
1196/**
1197 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list
1198 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1199 * @...: Variable number of arg (pairs), one pair for each field
1200 *
1201 * NOTE: Users normally won't want to call this function directly, but
1202 * rather use the kprobe_event_add_fields() wrapper, which
1203 * automatically adds a NULL to the end of the arg list.  If this
1204 * function is used directly, make sure the last arg in the variable
1205 * arg list is NULL.
1206 *
1207 * Add probe fields to an existing kprobe command using a variable
1208 * list of args.  Fields are added in the same order they're listed.
1209 *
1210 * Return: 0 if successful, error otherwise.
1211 */
1212int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...)
1213{
1214	struct dynevent_arg arg;
1215	va_list args;
1216	int ret = 0;
1217
1218	if (cmd->type != DYNEVENT_TYPE_KPROBE)
1219		return -EINVAL;
1220
1221	dynevent_arg_init(&arg, 0);
1222
1223	va_start(args, cmd);
1224	for (;;) {
1225		const char *field;
1226
1227		field = va_arg(args, const char *);
1228		if (!field)
1229			break;
1230
1231		if (++cmd->n_fields > MAX_TRACE_ARGS) {
1232			ret = -EINVAL;
1233			break;
1234		}
1235
1236		arg.str = field;
1237		ret = dynevent_arg_add(cmd, &arg, NULL);
1238		if (ret)
1239			break;
 
 
1240	}
1241	va_end(args);
1242
1243	return ret;
1244}
1245EXPORT_SYMBOL_GPL(__kprobe_event_add_fields);
1246
1247/**
1248 * kprobe_event_delete - Delete a kprobe event
1249 * @name: The name of the kprobe event to delete
1250 *
1251 * Delete a kprobe event with the give @name from kernel code rather
1252 * than directly from the command line.
1253 *
1254 * Return: 0 if successful, error otherwise.
1255 */
1256int kprobe_event_delete(const char *name)
1257{
1258	char buf[MAX_EVENT_NAME_LEN];
1259
1260	snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name);
1261
1262	return create_or_delete_trace_kprobe(buf);
1263}
1264EXPORT_SYMBOL_GPL(kprobe_event_delete);
1265
1266static int trace_kprobe_release(struct dyn_event *ev)
1267{
1268	struct trace_kprobe *tk = to_trace_kprobe(ev);
1269	int ret = unregister_trace_kprobe(tk);
1270
1271	if (!ret)
1272		free_trace_kprobe(tk);
1273	return ret;
1274}
1275
1276static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
1277{
1278	struct trace_kprobe *tk = to_trace_kprobe(ev);
1279	int i;
1280
1281	seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
1282	if (trace_kprobe_is_return(tk) && tk->rp.maxactive)
1283		seq_printf(m, "%d", tk->rp.maxactive);
1284	seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp),
1285				trace_probe_name(&tk->tp));
1286
1287	if (!tk->symbol)
1288		seq_printf(m, " 0x%p", tk->rp.kp.addr);
1289	else if (tk->rp.kp.offset)
1290		seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
1291			   tk->rp.kp.offset);
1292	else
1293		seq_printf(m, " %s", trace_kprobe_symbol(tk));
1294
1295	for (i = 0; i < tk->tp.nr_args; i++)
1296		seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
1297	seq_putc(m, '\n');
1298
1299	return 0;
1300}
1301
1302static int probes_seq_show(struct seq_file *m, void *v)
1303{
1304	struct dyn_event *ev = v;
1305
1306	if (!is_trace_kprobe(ev))
1307		return 0;
1308
1309	return trace_kprobe_show(m, ev);
1310}
1311
1312static const struct seq_operations probes_seq_op = {
1313	.start  = dyn_event_seq_start,
1314	.next   = dyn_event_seq_next,
1315	.stop   = dyn_event_seq_stop,
1316	.show   = probes_seq_show
1317};
1318
1319static int probes_open(struct inode *inode, struct file *file)
1320{
1321	int ret;
 
 
1322
1323	ret = security_locked_down(LOCKDOWN_TRACEFS);
1324	if (ret)
1325		return ret;
1326
1327	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1328		ret = dyn_events_release_all(&trace_kprobe_ops);
1329		if (ret < 0)
1330			return ret;
1331	}
1332
1333	return seq_open(file, &probes_seq_op);
 
 
 
 
 
 
 
 
1334}
1335
 
 
1336static ssize_t probes_write(struct file *file, const char __user *buffer,
1337			    size_t count, loff_t *ppos)
1338{
1339	return trace_parse_run_command(file, buffer, count, ppos,
1340				       create_or_delete_trace_kprobe);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1341}
1342
1343static const struct file_operations kprobe_events_ops = {
1344	.owner          = THIS_MODULE,
1345	.open           = probes_open,
1346	.read           = seq_read,
1347	.llseek         = seq_lseek,
1348	.release        = seq_release,
1349	.write		= probes_write,
1350};
1351
1352static unsigned long trace_kprobe_missed(struct trace_kprobe *tk)
1353{
1354	return trace_kprobe_is_return(tk) ?
1355		tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;
1356}
1357
1358/* Probes profiling interfaces */
1359static int probes_profile_seq_show(struct seq_file *m, void *v)
1360{
1361	struct dyn_event *ev = v;
1362	struct trace_kprobe *tk;
1363	unsigned long nmissed;
1364
1365	if (!is_trace_kprobe(ev))
1366		return 0;
1367
1368	tk = to_trace_kprobe(ev);
1369	nmissed = trace_kprobe_missed(tk);
1370	seq_printf(m, "  %-44s %15lu %15lu\n",
1371		   trace_probe_name(&tk->tp),
1372		   trace_kprobe_nhit(tk),
1373		   nmissed);
1374
1375	return 0;
1376}
1377
1378static const struct seq_operations profile_seq_op = {
1379	.start  = dyn_event_seq_start,
1380	.next   = dyn_event_seq_next,
1381	.stop   = dyn_event_seq_stop,
1382	.show   = probes_profile_seq_show
1383};
1384
1385static int profile_open(struct inode *inode, struct file *file)
1386{
1387	int ret;
1388
1389	ret = security_locked_down(LOCKDOWN_TRACEFS);
1390	if (ret)
1391		return ret;
1392
1393	return seq_open(file, &profile_seq_op);
1394}
1395
1396static const struct file_operations kprobe_profile_ops = {
1397	.owner          = THIS_MODULE,
1398	.open           = profile_open,
1399	.read           = seq_read,
1400	.llseek         = seq_lseek,
1401	.release        = seq_release,
1402};
1403
1404/* Note that we don't verify it, since the code does not come from user space */
1405static int
1406process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
1407		   void *dest, void *base)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1408{
1409	struct pt_regs *regs = rec;
1410	unsigned long val;
1411	int ret;
1412
1413retry:
1414	/* 1st stage: get value from context */
1415	switch (code->op) {
1416	case FETCH_OP_REG:
1417		val = regs_get_register(regs, code->param);
1418		break;
1419	case FETCH_OP_STACK:
1420		val = regs_get_kernel_stack_nth(regs, code->param);
1421		break;
1422	case FETCH_OP_STACKP:
1423		val = kernel_stack_pointer(regs);
1424		break;
1425	case FETCH_OP_RETVAL:
1426		val = regs_return_value(regs);
1427		break;
1428#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1429	case FETCH_OP_ARG:
1430		val = regs_get_kernel_argument(regs, code->param);
1431		break;
1432	case FETCH_OP_EDATA:
1433		val = *(unsigned long *)((unsigned long)edata + code->offset);
1434		break;
1435#endif
1436	case FETCH_NOP_SYMBOL:	/* Ignore a place holder */
1437		code++;
1438		goto retry;
1439	default:
1440		ret = process_common_fetch_insn(code, &val);
1441		if (ret < 0)
1442			return ret;
1443	}
1444	code++;
1445
1446	return process_fetch_insn_bottom(code, val, dest, base);
1447}
1448NOKPROBE_SYMBOL(process_fetch_insn)
1449
1450/* Kprobe handler */
1451static nokprobe_inline void
1452__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
1453		    struct trace_event_file *trace_file)
1454{
 
1455	struct kprobe_trace_entry_head *entry;
1456	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1457	struct trace_event_buffer fbuffer;
1458	int dsize;
1459
1460	WARN_ON(call != trace_file->event_call);
1461
1462	if (trace_trigger_soft_disabled(trace_file))
 
 
 
 
 
 
 
 
 
 
1463		return;
1464
1465	dsize = __get_data_size(&tk->tp, regs, NULL);
1466
1467	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
1468					   sizeof(*entry) + tk->tp.size + dsize);
1469	if (!entry)
1470		return;
1471
1472	fbuffer.regs = regs;
1473	entry->ip = (unsigned long)tk->rp.kp.addr;
1474	store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);
1475
1476	trace_event_buffer_commit(&fbuffer);
1477}
1478
1479static void
1480kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
1481{
1482	struct event_file_link *link;
1483
1484	trace_probe_for_each_link_rcu(link, &tk->tp)
1485		__kprobe_trace_func(tk, regs, link->file);
1486}
1487NOKPROBE_SYMBOL(kprobe_trace_func);
1488
1489/* Kretprobe handler */
1490
1491static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,
1492				      struct pt_regs *regs)
1493{
1494	struct kretprobe *rp = get_kretprobe(ri);
1495	struct trace_kprobe *tk;
1496
1497	/*
1498	 * There is a small chance that get_kretprobe(ri) returns NULL when
1499	 * the kretprobe is unregister on another CPU between kretprobe's
1500	 * trampoline_handler and this function.
1501	 */
1502	if (unlikely(!rp))
1503		return -ENOENT;
1504
1505	tk = container_of(rp, struct trace_kprobe, rp);
1506
1507	/* store argument values into ri->data as entry data */
1508	if (tk->tp.entry_arg)
1509		store_trace_entry_data(ri->data, &tk->tp, regs);
1510
1511	return 0;
1512}
1513
1514
1515static nokprobe_inline void
1516__kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1517		       struct pt_regs *regs,
1518		       struct trace_event_file *trace_file)
1519{
 
1520	struct kretprobe_trace_entry_head *entry;
1521	struct trace_event_buffer fbuffer;
1522	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1523	int dsize;
1524
1525	WARN_ON(call != trace_file->event_call);
1526
1527	if (trace_trigger_soft_disabled(trace_file))
 
 
 
 
 
 
 
 
1528		return;
1529
1530	dsize = __get_data_size(&tk->tp, regs, ri->data);
1531
1532	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
1533					   sizeof(*entry) + tk->tp.size + dsize);
1534	if (!entry)
1535		return;
1536
1537	fbuffer.regs = regs;
1538	entry->func = (unsigned long)tk->rp.kp.addr;
1539	entry->ret_ip = get_kretprobe_retaddr(ri);
1540	store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize);
1541
1542	trace_event_buffer_commit(&fbuffer);
1543}
1544
1545static void
1546kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1547		     struct pt_regs *regs)
1548{
1549	struct event_file_link *link;
1550
1551	trace_probe_for_each_link_rcu(link, &tk->tp)
1552		__kretprobe_trace_func(tk, ri, regs, link->file);
1553}
1554NOKPROBE_SYMBOL(kretprobe_trace_func);
1555
1556/* Event entry printers */
1557static enum print_line_t
1558print_kprobe_event(struct trace_iterator *iter, int flags,
1559		   struct trace_event *event)
1560{
1561	struct kprobe_trace_entry_head *field;
1562	struct trace_seq *s = &iter->seq;
1563	struct trace_probe *tp;
 
 
1564
1565	field = (struct kprobe_trace_entry_head *)iter->ent;
1566	tp = trace_probe_primary_from_call(
1567		container_of(event, struct trace_event_call, event));
1568	if (WARN_ON_ONCE(!tp))
1569		goto out;
1570
1571	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
 
1572
1573	if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
1574		goto out;
 
 
 
1575
1576	trace_seq_putc(s, ')');
 
 
 
 
1577
1578	if (trace_probe_print_args(s, tp->args, tp->nr_args,
1579			     (u8 *)&field[1], field) < 0)
1580		goto out;
1581
1582	trace_seq_putc(s, '\n');
1583 out:
1584	return trace_handle_return(s);
1585}
1586
1587static enum print_line_t
1588print_kretprobe_event(struct trace_iterator *iter, int flags,
1589		      struct trace_event *event)
1590{
1591	struct kretprobe_trace_entry_head *field;
1592	struct trace_seq *s = &iter->seq;
1593	struct trace_probe *tp;
 
 
1594
1595	field = (struct kretprobe_trace_entry_head *)iter->ent;
1596	tp = trace_probe_primary_from_call(
1597		container_of(event, struct trace_event_call, event));
1598	if (WARN_ON_ONCE(!tp))
1599		goto out;
1600
1601	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
 
1602
1603	if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
1604		goto out;
1605
1606	trace_seq_puts(s, " <- ");
 
1607
1608	if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
1609		goto out;
1610
1611	trace_seq_putc(s, ')');
 
1612
1613	if (trace_probe_print_args(s, tp->args, tp->nr_args,
1614			     (u8 *)&field[1], field) < 0)
1615		goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1616
1617	trace_seq_putc(s, '\n');
 
 
 
 
1618
1619 out:
1620	return trace_handle_return(s);
 
 
 
 
 
 
 
 
 
 
 
1621}
1622
 
 
 
 
 
1623
1624static int kprobe_event_define_fields(struct trace_event_call *event_call)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1625{
1626	int ret;
1627	struct kprobe_trace_entry_head field;
1628	struct trace_probe *tp;
1629
1630	tp = trace_probe_primary_from_call(event_call);
1631	if (WARN_ON_ONCE(!tp))
1632		return -ENOENT;
1633
1634	DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1635
1636	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1637}
1638
1639static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1640{
1641	int ret;
1642	struct kretprobe_trace_entry_head field;
1643	struct trace_probe *tp;
1644
1645	tp = trace_probe_primary_from_call(event_call);
1646	if (WARN_ON_ONCE(!tp))
1647		return -ENOENT;
 
 
1648
1649	DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1650	DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
 
1651
1652	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1653}
1654
1655#ifdef CONFIG_PERF_EVENTS
1656
1657/* Kprobe profile handler */
1658static int
1659kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1660{
1661	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
 
1662	struct kprobe_trace_entry_head *entry;
1663	struct hlist_head *head;
1664	int size, __size, dsize;
1665	int rctx;
1666
1667	if (bpf_prog_array_valid(call)) {
1668		unsigned long orig_ip = instruction_pointer(regs);
1669		int ret;
1670
1671		ret = trace_call_bpf(call, regs);
1672
1673		/*
1674		 * We need to check and see if we modified the pc of the
1675		 * pt_regs, and if so return 1 so that we don't do the
1676		 * single stepping.
1677		 */
1678		if (orig_ip != instruction_pointer(regs))
1679			return 1;
1680		if (!ret)
1681			return 0;
1682	}
1683
1684	head = this_cpu_ptr(call->perf_events);
1685	if (hlist_empty(head))
1686		return 0;
1687
1688	dsize = __get_data_size(&tk->tp, regs, NULL);
1689	__size = sizeof(*entry) + tk->tp.size + dsize;
1690	size = ALIGN(__size + sizeof(u32), sizeof(u64));
1691	size -= sizeof(u32);
 
 
 
1692
1693	entry = perf_trace_buf_alloc(size, NULL, &rctx);
1694	if (!entry)
1695		return 0;
1696
1697	entry->ip = (unsigned long)tk->rp.kp.addr;
1698	memset(&entry[1], 0, dsize);
1699	store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);
1700	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1701			      head, NULL);
1702	return 0;
1703}
1704NOKPROBE_SYMBOL(kprobe_perf_func);
1705
1706/* Kretprobe profile handler */
1707static void
1708kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1709		    struct pt_regs *regs)
1710{
1711	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
 
1712	struct kretprobe_trace_entry_head *entry;
1713	struct hlist_head *head;
1714	int size, __size, dsize;
1715	int rctx;
1716
1717	if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1718		return;
1719
1720	head = this_cpu_ptr(call->perf_events);
1721	if (hlist_empty(head))
1722		return;
1723
1724	dsize = __get_data_size(&tk->tp, regs, ri->data);
1725	__size = sizeof(*entry) + tk->tp.size + dsize;
1726	size = ALIGN(__size + sizeof(u32), sizeof(u64));
1727	size -= sizeof(u32);
 
 
 
1728
1729	entry = perf_trace_buf_alloc(size, NULL, &rctx);
1730	if (!entry)
1731		return;
1732
1733	entry->func = (unsigned long)tk->rp.kp.addr;
1734	entry->ret_ip = get_kretprobe_retaddr(ri);
1735	store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize);
1736	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1737			      head, NULL);
1738}
1739NOKPROBE_SYMBOL(kretprobe_perf_func);
1740
1741int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
1742			const char **symbol, u64 *probe_offset,
1743			u64 *probe_addr, unsigned long *missed,
1744			bool perf_type_tracepoint)
1745{
1746	const char *pevent = trace_event_name(event->tp_event);
1747	const char *group = event->tp_event->class->system;
1748	struct trace_kprobe *tk;
1749
1750	if (perf_type_tracepoint)
1751		tk = find_trace_kprobe(pevent, group);
1752	else
1753		tk = trace_kprobe_primary_from_call(event->tp_event);
1754	if (!tk)
1755		return -EINVAL;
1756
1757	*fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE
1758					      : BPF_FD_TYPE_KPROBE;
1759	*probe_offset = tk->rp.kp.offset;
1760	*probe_addr = kallsyms_show_value(current_cred()) ?
1761		      (unsigned long)tk->rp.kp.addr : 0;
1762	*symbol = tk->symbol;
1763	if (missed)
1764		*missed = trace_kprobe_missed(tk);
1765	return 0;
1766}
1767#endif	/* CONFIG_PERF_EVENTS */
1768
1769/*
1770 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1771 *
1772 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1773 * lockless, but we can't race with this __init function.
1774 */
1775static int kprobe_register(struct trace_event_call *event,
1776			   enum trace_reg type, void *data)
1777{
1778	struct trace_event_file *file = data;
1779
1780	switch (type) {
1781	case TRACE_REG_REGISTER:
1782		return enable_trace_kprobe(event, file);
1783	case TRACE_REG_UNREGISTER:
1784		return disable_trace_kprobe(event, file);
 
1785
1786#ifdef CONFIG_PERF_EVENTS
1787	case TRACE_REG_PERF_REGISTER:
1788		return enable_trace_kprobe(event, NULL);
1789	case TRACE_REG_PERF_UNREGISTER:
1790		return disable_trace_kprobe(event, NULL);
1791	case TRACE_REG_PERF_OPEN:
1792	case TRACE_REG_PERF_CLOSE:
1793	case TRACE_REG_PERF_ADD:
1794	case TRACE_REG_PERF_DEL:
1795		return 0;
1796#endif
1797	}
1798	return 0;
1799}
1800
1801static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
 
1802{
1803	struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1804	int ret = 0;
1805
1806	raw_cpu_inc(*tk->nhit);
1807
1808	if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))
1809		kprobe_trace_func(tk, regs);
1810#ifdef CONFIG_PERF_EVENTS
1811	if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))
1812		ret = kprobe_perf_func(tk, regs);
1813#endif
1814	return ret;
1815}
1816NOKPROBE_SYMBOL(kprobe_dispatcher);
1817
1818static int
1819kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1820{
1821	struct kretprobe *rp = get_kretprobe(ri);
1822	struct trace_kprobe *tk;
1823
1824	/*
1825	 * There is a small chance that get_kretprobe(ri) returns NULL when
1826	 * the kretprobe is unregister on another CPU between kretprobe's
1827	 * trampoline_handler and this function.
1828	 */
1829	if (unlikely(!rp))
1830		return 0;
1831
1832	tk = container_of(rp, struct trace_kprobe, rp);
1833	raw_cpu_inc(*tk->nhit);
1834
1835	if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))
1836		kretprobe_trace_func(tk, ri, regs);
1837#ifdef CONFIG_PERF_EVENTS
1838	if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))
1839		kretprobe_perf_func(tk, ri, regs);
1840#endif
1841	return 0;	/* We don't tweak kernel, so just return 0 */
1842}
1843NOKPROBE_SYMBOL(kretprobe_dispatcher);
1844
1845static struct trace_event_functions kretprobe_funcs = {
1846	.trace		= print_kretprobe_event
1847};
1848
1849static struct trace_event_functions kprobe_funcs = {
1850	.trace		= print_kprobe_event
1851};
1852
1853static struct trace_event_fields kretprobe_fields_array[] = {
1854	{ .type = TRACE_FUNCTION_TYPE,
1855	  .define_fields = kretprobe_event_define_fields },
1856	{}
1857};
1858
1859static struct trace_event_fields kprobe_fields_array[] = {
1860	{ .type = TRACE_FUNCTION_TYPE,
1861	  .define_fields = kprobe_event_define_fields },
1862	{}
1863};
1864
1865static inline void init_trace_event_call(struct trace_kprobe *tk)
1866{
1867	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
 
1868
1869	if (trace_kprobe_is_return(tk)) {
 
 
1870		call->event.funcs = &kretprobe_funcs;
1871		call->class->fields_array = kretprobe_fields_array;
1872	} else {
1873		call->event.funcs = &kprobe_funcs;
1874		call->class->fields_array = kprobe_fields_array;
1875	}
1876
1877	call->flags = TRACE_EVENT_FL_KPROBE;
1878	call->class->reg = kprobe_register;
1879}
1880
1881static int register_kprobe_event(struct trace_kprobe *tk)
1882{
1883	init_trace_event_call(tk);
1884
1885	return trace_probe_register_event_call(&tk->tp);
1886}
1887
1888static int unregister_kprobe_event(struct trace_kprobe *tk)
1889{
1890	return trace_probe_unregister_event_call(&tk->tp);
1891}
1892
1893#ifdef CONFIG_PERF_EVENTS
1894
1895/* create a trace_kprobe, but don't add it to global lists */
1896struct trace_event_call *
1897create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1898			  bool is_return)
1899{
1900	enum probe_print_type ptype;
1901	struct trace_kprobe *tk;
1902	int ret;
1903	char *event;
1904
1905	if (func) {
1906		ret = validate_probe_symbol(func);
1907		if (ret)
1908			return ERR_PTR(ret);
1909	}
1910
1911	/*
1912	 * local trace_kprobes are not added to dyn_event, so they are never
1913	 * searched in find_trace_kprobe(). Therefore, there is no concern of
1914	 * duplicated name here.
1915	 */
1916	event = func ? func : "DUMMY_EVENT";
1917
1918	tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1919				offs, 0 /* maxactive */, 0 /* nargs */,
1920				is_return);
1921
1922	if (IS_ERR(tk)) {
1923		pr_info("Failed to allocate trace_probe.(%d)\n",
1924			(int)PTR_ERR(tk));
1925		return ERR_CAST(tk);
1926	}
1927
1928	init_trace_event_call(tk);
1929
1930	ptype = trace_kprobe_is_return(tk) ?
1931		PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1932	if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) {
1933		ret = -ENOMEM;
1934		goto error;
1935	}
1936
1937	ret = __register_trace_kprobe(tk);
1938	if (ret < 0)
1939		goto error;
1940
1941	return trace_probe_event_call(&tk->tp);
1942error:
1943	free_trace_kprobe(tk);
1944	return ERR_PTR(ret);
1945}
1946
1947void destroy_local_trace_kprobe(struct trace_event_call *event_call)
1948{
1949	struct trace_kprobe *tk;
1950
1951	tk = trace_kprobe_primary_from_call(event_call);
1952	if (unlikely(!tk))
1953		return;
1954
1955	if (trace_probe_is_enabled(&tk->tp)) {
1956		WARN_ON(1);
1957		return;
1958	}
1959
1960	__unregister_trace_kprobe(tk);
1961
1962	free_trace_kprobe(tk);
1963}
1964#endif /* CONFIG_PERF_EVENTS */
1965
1966static __init void enable_boot_kprobe_events(void)
1967{
1968	struct trace_array *tr = top_trace_array();
1969	struct trace_event_file *file;
1970	struct trace_kprobe *tk;
1971	struct dyn_event *pos;
1972
1973	mutex_lock(&event_mutex);
1974	for_each_trace_kprobe(tk, pos) {
1975		list_for_each_entry(file, &tr->events, list)
1976			if (file->event_call == trace_probe_event_call(&tk->tp))
1977				trace_event_enable_disable(file, 1, 0);
1978	}
1979	mutex_unlock(&event_mutex);
1980}
1981
1982static __init void setup_boot_kprobe_events(void)
1983{
1984	char *p, *cmd = kprobe_boot_events_buf;
1985	int ret;
1986
1987	strreplace(kprobe_boot_events_buf, ',', ' ');
1988
1989	while (cmd && *cmd != '\0') {
1990		p = strchr(cmd, ';');
1991		if (p)
1992			*p++ = '\0';
1993
1994		ret = create_or_delete_trace_kprobe(cmd);
1995		if (ret)
1996			pr_warn("Failed to add event(%d): %s\n", ret, cmd);
1997
1998		cmd = p;
1999	}
2000
2001	enable_boot_kprobe_events();
2002}
2003
2004/*
2005 * Register dynevent at core_initcall. This allows kernel to setup kprobe
2006 * events in postcore_initcall without tracefs.
2007 */
2008static __init int init_kprobe_trace_early(void)
2009{
2010	int ret;
2011
2012	ret = dyn_event_register(&trace_kprobe_ops);
2013	if (ret)
2014		return ret;
2015
2016	if (trace_kprobe_register_module_notifier())
2017		return -EINVAL;
2018
2019	return 0;
2020}
2021core_initcall(init_kprobe_trace_early);
2022
2023/* Make a tracefs interface for controlling probe points */
2024static __init int init_kprobe_trace(void)
2025{
2026	int ret;
2027
2028	ret = tracing_init_dentry();
2029	if (ret)
2030		return 0;
2031
 
 
 
2032	/* Event list interface */
2033	trace_create_file("kprobe_events", TRACE_MODE_WRITE,
2034			  NULL, NULL, &kprobe_events_ops);
 
2035
2036	/* Profile interface */
2037	trace_create_file("kprobe_profile", TRACE_MODE_READ,
2038			  NULL, NULL, &kprobe_profile_ops);
2039
2040	setup_boot_kprobe_events();
2041
 
 
 
2042	return 0;
2043}
2044fs_initcall(init_kprobe_trace);
2045
2046
2047#ifdef CONFIG_FTRACE_STARTUP_TEST
2048static __init struct trace_event_file *
2049find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
2050{
2051	struct trace_event_file *file;
2052
2053	list_for_each_entry(file, &tr->events, list)
2054		if (file->event_call == trace_probe_event_call(&tk->tp))
2055			return file;
2056
2057	return NULL;
2058}
2059
2060/*
2061 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
2062 * stage, we can do this lockless.
2063 */
 
 
 
 
 
 
2064static __init int kprobe_trace_self_tests_init(void)
2065{
2066	int ret, warn = 0;
2067	int (*target)(int, int, int, int, int, int);
2068	struct trace_kprobe *tk;
2069	struct trace_event_file *file;
2070
2071	if (tracing_is_disabled())
2072		return -ENODEV;
2073
2074	if (tracing_selftest_disabled)
2075		return 0;
2076
2077	target = kprobe_trace_selftest_target;
2078
2079	pr_info("Testing kprobe tracing: ");
2080
2081	ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)");
2082	if (WARN_ONCE(ret, "error on probing function entry.")) {
 
 
2083		warn++;
2084	} else {
2085		/* Enable trace point */
2086		tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2087		if (WARN_ONCE(tk == NULL, "error on probing function entry.")) {
 
2088			warn++;
2089		} else {
2090			file = find_trace_probe_file(tk, top_trace_array());
2091			if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2092				warn++;
2093			} else
2094				enable_trace_kprobe(
2095					trace_probe_event_call(&tk->tp), file);
2096		}
2097	}
2098
2099	ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval");
2100	if (WARN_ONCE(ret, "error on probing function return.")) {
 
 
2101		warn++;
2102	} else {
2103		/* Enable trace point */
2104		tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2105		if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) {
 
2106			warn++;
2107		} else {
2108			file = find_trace_probe_file(tk, top_trace_array());
2109			if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2110				warn++;
2111			} else
2112				enable_trace_kprobe(
2113					trace_probe_event_call(&tk->tp), file);
2114		}
2115	}
2116
2117	if (warn)
2118		goto end;
2119
2120	ret = target(1, 2, 3, 4, 5, 6);
2121
2122	/*
2123	 * Not expecting an error here, the check is only to prevent the
2124	 * optimizer from removing the call to target() as otherwise there
2125	 * are no side-effects and the call is never performed.
2126	 */
2127	if (ret != 21)
2128		warn++;
2129
2130	/* Disable trace points before removing it */
2131	tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2132	if (WARN_ONCE(tk == NULL, "error on getting test probe.")) {
2133		warn++;
2134	} else {
2135		if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
2136				 "incorrect number of testprobe hits."))
2137			warn++;
2138
2139		file = find_trace_probe_file(tk, top_trace_array());
2140		if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2141			warn++;
2142		} else
2143			disable_trace_kprobe(
2144				trace_probe_event_call(&tk->tp), file);
2145	}
2146
2147	tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2148	if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) {
 
2149		warn++;
2150	} else {
2151		if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
2152				 "incorrect number of testprobe2 hits."))
2153			warn++;
2154
2155		file = find_trace_probe_file(tk, top_trace_array());
2156		if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2157			warn++;
2158		} else
2159			disable_trace_kprobe(
2160				trace_probe_event_call(&tk->tp), file);
2161	}
2162
2163	ret = create_or_delete_trace_kprobe("-:testprobe");
2164	if (WARN_ONCE(ret, "error on deleting a probe."))
2165		warn++;
2166
2167	ret = create_or_delete_trace_kprobe("-:testprobe2");
2168	if (WARN_ONCE(ret, "error on deleting a probe."))
2169		warn++;
2170
2171
2172end:
2173	/*
2174	 * Wait for the optimizer work to finish. Otherwise it might fiddle
2175	 * with probes in already freed __init text.
2176	 */
2177	wait_for_kprobe_optimizer();
2178	if (warn)
2179		pr_cont("NG: Some tests are failed. Please check them.\n");
2180	else
2181		pr_cont("OK\n");
2182	return 0;
2183}
2184
2185late_initcall(kprobe_trace_self_tests_init);
2186
2187#endif